lychiyu/bbgo_origin
1
0
Fork 0
mirror of https://github.com/c9s/bbgo.git synced 2024-11-10 01:01:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

service: implement margin service for syncing margin related data

Browse Source
This commit is contained in:
c9s 2022-05-31 17:08:52 +08:00
parent 7601f08786
commit bf92e28461
No known key found for this signature in database
GPG Key ID: 7385E7E464CB0A54
14 changed files with 392 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -52,6 +52,7 @@ require (
)
require (
github.com/Masterminds/squirrel v1.5.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bitly/go-simplejson v0.5.0 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
@ -75,6 +76,8 @@ require (
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect
github.com/json-iterator/go v1.1.11 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lestrrat-go/strftime v1.0.0 // indirect
github.com/lib/pq v1.10.5 // indirect

6
go.sum
View File

@ -39,6 +39,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/Masterminds/squirrel v1.5.3 h1:YPpoceAcxuzIljlr5iWpNKaql7hLeG1KLSrhvdHpkZc=
github.com/Masterminds/squirrel v1.5.3/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/adshao/go-binance/v2 v2.3.5 h1:WVYZecm0w8l14YoWlnKZj6xxZT2AKMTHpMQSqIX1xxA=
github.com/adshao/go-binance/v2 v2.3.5/go.mod h1:8Pg/FGTLyAhq8QXA0IkoReKyRpoxJcK3LVujKDAZV/c=
@ -303,6 +305,10 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
github.com/leekchan/accounting v0.0.0-20191218023648-17a4ce5f94d4 h1:KZzDAtJ7ZLm0zSWVhN/zgyB8Ksx5H+P9irwbTcJ9FwI=
github.com/leekchan/accounting v0.0.0-20191218023648-17a4ce5f94d4/go.mod h1:3timm6YPhY3YDaGxl0q3eaflX0eoSx3FXn7ckHe4tO0=
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=

2
migrations/mysql/20220531013542_margin_interests.sql
View File

@ -3,8 +3,6 @@ CREATE TABLE `margin_interests`
(
`gid` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`transaction_id` BIGINT UNSIGNED NOT NULL,
`exchange` VARCHAR(24) NOT NULL DEFAULT '',
`asset` VARCHAR(24) NOT NULL DEFAULT '',

2
migrations/mysql/20220531015005_margin_liquidations.sql
View File

@ -23,7 +23,7 @@ CREATE TABLE `margin_liquidations`
`time_in_force` VARCHAR(5) NOT NULL DEFAULT '',
`updated_time` DATETIME(3) NOT NULL,
`time` DATETIME(3) NOT NULL,
PRIMARY KEY (`gid`),
UNIQUE KEY (`order_id`, `exchange`)

2
migrations/sqlite3/20220531013541_margin_interests.sql
View File

@ -3,8 +3,6 @@ CREATE TABLE `margin_interests`
(
`gid` INTEGER PRIMARY KEY AUTOINCREMENT,
`transaction_id` INTEGER NOT NULL,
`exchange` VARCHAR(24) NOT NULL DEFAULT '',
`asset` VARCHAR(24) NOT NULL DEFAULT '',

14
migrations/sqlite3/20220531015005_margin_liquidations.sql
View File

@ -3,13 +3,13 @@ CREATE TABLE `margin_liquidations`
(
`gid` INTEGER PRIMARY KEY AUTOINCREMENT,
`exchange` VARCHAR(24) NOT NULL DEFAULT '',
`exchange` VARCHAR(24) NOT NULL DEFAULT '',
`symbol` VARCHAR(24) NOT NULL DEFAULT '',
`symbol` VARCHAR(24) NOT NULL DEFAULT '',
`order_id` INTEGER NOT NULL,
`order_id` INTEGER NOT NULL,
`is_isolated` BOOL NOT NULL DEFAULT false,
`is_isolated` BOOL NOT NULL DEFAULT false,
`average_price` DECIMAL(16, 8) NOT NULL,
@ -19,11 +19,11 @@ CREATE TABLE `margin_liquidations`
`executed_quantity` DECIMAL(16, 8) NOT NULL,
`side` VARCHAR(5) NOT NULL DEFAULT '',
`side` VARCHAR(5) NOT NULL DEFAULT '',
`time_in_force` VARCHAR(5) NOT NULL DEFAULT '',
`time_in_force` VARCHAR(5) NOT NULL DEFAULT '',
`updated_time` DATETIME(3) NOT NULL
`time` DATETIME(3) NOT NULL
);
-- +down

36
pkg/exchange/batch/margin_liquidation.go Normal file
View File

@ -0,0 +1,36 @@
package batch
import (
"context"
"strconv"
"time"
"golang.org/x/time/rate"
"github.com/c9s/bbgo/pkg/types"
)
type MarginLiquidationBatchQuery struct {
types.MarginHistory
}
func (e *MarginLiquidationBatchQuery) Query(ctx context.Context, startTime, endTime time.Time) (c chan types.MarginLiquidation, errC chan error) {
query := &AsyncTimeRangedBatchQuery{
Type: types.MarginLiquidation{},
Limiter: rate.NewLimiter(rate.Every(5*time.Second), 2),
Q: func(startTime, endTime time.Time) (interface{}, error) {
return e.QueryLiquidationHistory(ctx, &startTime, &endTime)
},
T: func(obj interface{}) time.Time {
return time.Time(obj.(types.MarginLiquidation).UpdatedTime)
},
ID: func(obj interface{}) string {
liquidation := obj.(types.MarginLiquidation)
return strconv.FormatUint(liquidation.OrderID, 10)
},
}
c = make(chan types.MarginLiquidation, 100)
errC = query.Query(ctx, c, startTime, endTime)
return c, errC
}

2
pkg/exchange/batch/margin_loan.go
View File

@ -22,7 +22,7 @@ func (e *MarginLoanBatchQuery) Query(ctx context.Context, asset string, startTim
return e.QueryLoanHistory(ctx, asset, &startTime, &endTime)
},
T: func(obj interface{}) time.Time {
return time.Time(obj.(types.MarginRepay).Time)
return time.Time(obj.(types.MarginLoan).Time)
},
ID: func(obj interface{}) string {
loan := obj.(types.MarginLoan)

25
pkg/exchange/binance/binanceapi/client_test.go
View File

@ -4,32 +4,15 @@ import (
"context"
"log"
"net/http/httputil"
"os"
"regexp"
"testing"
"github.com/stretchr/testify/assert"
"github.com/c9s/bbgo/pkg/testutil"
)
func maskSecret(s string) string {
re := regexp.MustCompile(`\b(\w{4})\w+\b`)
s = re.ReplaceAllString(s, "$1******")
return s
}
func integrationTestConfigured(t *testing.T, prefix string) (key, secret string, ok bool) {
var hasKey, hasSecret bool
key, hasKey = os.LookupEnv(prefix + "_API_KEY")
secret, hasSecret = os.LookupEnv(prefix + "_API_SECRET")
ok = hasKey && hasSecret && os.Getenv("TEST_"+prefix) == "1"
if ok {
t.Logf(prefix+" api integration test enabled, key = %s, secret = %s", maskSecret(key), maskSecret(secret))
}
return key, secret, ok
}
func getTestClientOrSkip(t *testing.T) *RestClient {
key, secret, ok := integrationTestConfigured(t, "BINANCE")
key, secret, ok := testutil.IntegrationTestConfigured(t, "BINANCE")
if !ok {
t.SkipNow()
return nil
@ -119,7 +102,7 @@ func TestClient_NewGetMarginInterestRateHistoryRequest(t *testing.T) {
}
func TestClient_privateCall(t *testing.T) {
key, secret, ok := integrationTestConfigured(t, "BINANCE")
key, secret, ok := testutil.IntegrationTestConfigured(t, "BINANCE")
if !ok {
t.SkipNow()
}

260
pkg/service/margin.go Normal file
View File

@ -0,0 +1,260 @@
package service
import (
"context"
"fmt"
"reflect"
"sort"
"strconv"
"time"
sq "github.com/Masterminds/squirrel"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
"github.com/c9s/bbgo/pkg/exchange/batch"
"github.com/c9s/bbgo/pkg/types"
)
// SyncSelect defines the behaviors for syncing remote records
type SyncSelect struct {
Select sq.SelectBuilder
Type interface{}
BatchQuery func(ctx context.Context, startTime, endTime time.Time) (interface{}, chan error)
// ID is a function that returns the unique identity of the object
ID func(obj interface{}) string
// Time is a function that returns the time of the object
Time func(obj interface{}) time.Time
}
type MarginService struct {
DB *sqlx.DB
}
func (s *MarginService) Sync(ctx context.Context, ex types.Exchange, asset string, startTime time.Time) error {
api, ok := ex.(types.MarginHistory)
if !ok {
return ErrNotImplemented
}
marginExchange, ok := ex.(types.MarginExchange)
if !ok {
return fmt.Errorf("%T does not implement margin service", ex)
}
marginSettings := marginExchange.GetMarginSettings()
if !marginSettings.IsMargin {
return fmt.Errorf("exchange instance %s is not using margin", ex.Name())
}
sels := []SyncSelect{
{
Select: SelectLastMarginLoans(ex.Name(), 100),
Type: types.MarginLoan{},
BatchQuery: func(ctx context.Context, startTime, endTime time.Time) (interface{}, chan error) {
query := &batch.MarginLoanBatchQuery{
MarginHistory: api,
}
return query.Query(ctx, asset, startTime, endTime)
},
ID: func(obj interface{}) string {
return strconv.FormatUint(obj.(types.MarginLoan).TransactionID, 10)
},
},
{
Select: SelectLastMarginRepays(ex.Name(), 100),
Type: types.MarginRepay{},
BatchQuery: func(ctx context.Context, startTime, endTime time.Time) (interface{}, chan error) {
query := &batch.MarginRepayBatchQuery{
MarginHistory: api,
}
return query.Query(ctx, asset, startTime, endTime)
},
ID: func(obj interface{}) string {
return strconv.FormatUint(obj.(types.MarginRepay).TransactionID, 10)
},
},
{
Select: SelectLastMarginInterests(ex.Name(), 100),
Type: types.MarginInterest{},
BatchQuery: func(ctx context.Context, startTime, endTime time.Time) (interface{}, chan error) {
query := &batch.MarginInterestBatchQuery{
MarginHistory: api,
}
return query.Query(ctx, asset, startTime, endTime)
},
ID: func(obj interface{}) string {
m := obj.(types.MarginInterest)
return m.Asset + m.IsolatedSymbol + strconv.FormatInt(m.Time.UnixMilli(), 10)
},
},
}
NextQuery:
for _, sel := range sels {
// query from db
recordSlice, err := s.executeDbQuery(ctx, sel.Select, sel.Type)
if err != nil {
return err
}
recordSliceRef := reflect.ValueOf(recordSlice)
if recordSliceRef.Kind() == reflect.Ptr {
recordSliceRef = recordSliceRef.Elem()
}
logrus.Debugf("loaded %d records", recordSliceRef.Len())
ids := buildIdMap(sel, recordSliceRef)
sortRecords(sel, recordSliceRef)
// default since time point
since := lastRecordTime(sel, recordSliceRef, startTime)
// asset "" means all assets
dataC, errC := sel.BatchQuery(ctx, since, time.Now())
dataCRef := reflect.ValueOf(dataC)
for {
select {
case <-ctx.Done():
return nil
case err := <-errC:
return err
default:
v, ok := dataCRef.Recv()
if !ok {
err := <-errC
if err != nil {
return err
}
// closed chan, skip to next query
continue NextQuery
}
obj := v.Interface()
id := sel.ID(obj)
if _, exists := ids[id]; exists {
continue
}
logrus.Debugf("inserting %T: %+v", obj, obj)
if err := s.Insert(obj); err != nil {
return err
}
}
}
}
return nil
}
func (s *MarginService) executeDbQuery(ctx context.Context, sel sq.SelectBuilder, tpe interface{}) (interface{}, error) {
sql, args, err := sel.ToSql()
if err != nil {
return nil, err
}
rows, err := s.DB.QueryxContext(ctx, sql, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return s.scanRows(rows, tpe)
}
func (s *MarginService) scanRows(rows *sqlx.Rows, tpe interface{}) (interface{}, error) {
refType := reflect.TypeOf(tpe)
if refType.Kind() == reflect.Ptr {
refType = refType.Elem()
}
sliceRef := reflect.New(reflect.SliceOf(refType))
for rows.Next() {
var recordRef = reflect.New(refType)
var record = recordRef.Interface()
if err := rows.StructScan(&record); err != nil {
return sliceRef.Interface(), err
}
sliceRef = reflect.Append(sliceRef, recordRef)
}
return sliceRef.Interface(), rows.Err()
}
func (s *MarginService) Insert(record interface{}) error {
sql := dbCache.InsertSqlOf(record)
_, err := s.DB.NamedExec(sql, record)
return err
}
func SelectLastMarginLoans(ex types.ExchangeName, limit uint64) sq.SelectBuilder {
return sq.Select("*").
From("margin_loans").
Where(sq.Eq{"exchange": ex}).
OrderBy("time").
Limit(limit)
}
func SelectLastMarginRepays(ex types.ExchangeName, limit uint64) sq.SelectBuilder {
return sq.Select("*").
From("margin_repays").
Where(sq.Eq{"exchange": ex}).
OrderBy("time").
Limit(limit)
}
func SelectLastMarginInterests(ex types.ExchangeName, limit uint64) sq.SelectBuilder {
return sq.Select("*").
From("margin_interests").
Where(sq.Eq{"exchange": ex}).
OrderBy("time").
Limit(limit)
}
func SelectLastMarginLiquidations(ex types.ExchangeName, limit uint64) sq.SelectBuilder {
return sq.Select("*").
From("margin_liquidations").
Where(sq.Eq{"exchange": ex}).
OrderBy("time").
Limit(limit)
}
func lastRecordTime(sel SyncSelect, recordSlice reflect.Value, defaultTime time.Time) time.Time {
since := defaultTime
length := recordSlice.Len()
if length > 0 {
since = sel.Time(recordSlice.Index(length - 1))
}
return since
}
func sortRecords(sel SyncSelect, recordSlice reflect.Value) {
// always sort
sort.Slice(recordSlice.Interface(), func(i, j int) bool {
a := sel.Time(recordSlice.Index(i).Interface())
b := sel.Time(recordSlice.Index(j).Interface())
return a.Before(b)
})
}
func buildIdMap(sel SyncSelect, recordSliceRef reflect.Value) map[string]struct{} {
ids := map[string]struct{}{}
for i := 0; i < recordSliceRef.Len(); i++ {
entryRef := recordSliceRef.Index(i)
id := sel.ID(entryRef.Interface())
ids[id] = struct{}{}
}
return ids
}

47
pkg/service/margin_test.go Normal file
View File

@ -0,0 +1,47 @@
package service
import (
"context"
"testing"
"time"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/c9s/bbgo/pkg/exchange/binance"
"github.com/c9s/bbgo/pkg/testutil"
)
func TestMarginService(t *testing.T) {
key, secret, ok := testutil.IntegrationTestConfigured(t, "BINANCE")
if !ok {
t.SkipNow()
return
}
ex := binance.New(key, secret)
ex.MarginSettings.IsMargin = true
ex.MarginSettings.IsIsolatedMargin = true
ex.MarginSettings.IsolatedMarginSymbol = "DOTUSDT"
db, err := prepareDB(t)
if err != nil {
t.Fatal(err)
}
defer db.Close()
ctx := context.Background()
logrus.SetLevel(logrus.DebugLevel)
dbx := sqlx.NewDb(db.DB, "sqlite3")
service := &MarginService{DB: dbx}
err = service.Sync(ctx, ex, "USDT", time.Date(2022, time.February, 1, 0, 0, 0, 0, time.UTC))
assert.NoError(t, err)
// sync second time to ensure that we can query records
err = service.Sync(ctx, ex, "USDT", time.Date(2022, time.February, 1, 0, 0, 0, 0, time.UTC))
assert.NoError(t, err)
}

2
pkg/service/reflect_test.go
View File

@ -58,7 +58,7 @@ func Test_fieldsNamesOf(t *testing.T) {
{
name: "MarginInterest",
args: args{record: &types.MarginInterest{}},
want: []string{"asset", "principle", "interest", "interest_rate", "isolated_symbol", "time"},
want: []string{"exchange", "asset", "principle", "interest", "interest_rate", "isolated_symbol", "time"},
},
}
for _, tt := range tests {

25
pkg/testutil/auth.go Normal file
View File

@ -0,0 +1,25 @@
package testutil
import (
"os"
"regexp"
"testing"
)
func maskSecret(s string) string {
re := regexp.MustCompile(`\b(\w{4})\w+\b`)
s = re.ReplaceAllString(s, "$1******")
return s
}
func IntegrationTestConfigured(t *testing.T, prefix string) (key, secret string, ok bool) {
var hasKey, hasSecret bool
key, hasKey = os.LookupEnv(prefix + "_API_KEY")
secret, hasSecret = os.LookupEnv(prefix + "_API_SECRET")
ok = hasKey && hasSecret && os.Getenv("TEST_"+prefix) == "1"
if ok {
t.Logf(prefix+" api integration test enabled, key = %s, secret = %s", maskSecret(key), maskSecret(secret))
}
return key, secret, ok
}

2
pkg/types/margin.go
View File

@ -98,7 +98,7 @@ type MarginLiquidation struct {
Symbol string `json:"symbol" db:"symbol"`
TimeInForce TimeInForce `json:"timeInForce" db:"time_in_force"`
IsIsolated bool `json:"isIsolated" db:"is_isolated"`
UpdatedTime Time `json:"updatedTime" db:"updated_time"`
UpdatedTime Time `json:"updatedTime" db:"time"`
}
// MarginHistory provides the service of querying loan history and repay history