lychiyu/bbgo_origin
1
0
Fork 0
mirror of https://github.com/c9s/bbgo.git synced 2024-11-10 09:11:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

database: sqlite3 issue fix

Browse Source 
database: upgrade gosqllite3 version for increasing variable amount limit
	types: update kline starttime/endtime field to prevent sqlite3
	time parsing issue.

	fix #215
This commit is contained in:
TonyQ 2021-12-15 13:04:01 +08:00
parent 3266571698
commit 3e45035ab1
28 changed files with 61 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -30,7 +30,7 @@ require (
github.com/magefile/mage v1.11.0 // indirect
github.com/magiconair/properties v1.8.4 // indirect
github.com/mattn/go-colorable v0.1.7 // indirect
github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
github.com/mattn/go-sqlite3 v1.14.9 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/pkg/errors v0.9.1

2
go.sum
View File

@ -246,6 +246,8 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=

2
pkg/backtest/exchange.go
View File

@ -266,7 +266,7 @@ func (e Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticker
kline := matching.LastKLine
return &types.Ticker{
Time: kline.EndTime,
Time: kline.EndTime.Time(),
Volume: kline.Volume,
Last: kline.Close,
Open: kline.Open,

2
pkg/backtest/matching.go
View File

@ -399,7 +399,7 @@ func (m *SimplePriceMatching) SellToPrice(price fixedpoint.Value) (closedOrders
}
func (m *SimplePriceMatching) processKLine(kline types.KLine) {
m.CurrentTime = kline.EndTime
m.CurrentTime = kline.EndTime.Time()
m.LastKLine = kline
switch kline.Direction() {

2
pkg/exchange/batch/batch.go
View File

@ -128,7 +128,7 @@ func (e KLineBatchQuery) Query(ctx context.Context, symbol string, interval type
}
//The issue is in FTX, prev endtime = next start time , so if add 1 ms , it would query forever.
currentTime = kline.StartTime
currentTime = kline.StartTime.Time()
tryQueryKlineTimes = 0
}

6
pkg/exchange/binance/exchange.go
View File

@ -919,8 +919,8 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type
Exchange: types.ExchangeBinance,
Symbol: symbol,
Interval: interval,
StartTime: time.Unix(0, k.OpenTime*int64(time.Millisecond)),
EndTime: time.Unix(0, k.CloseTime*int64(time.Millisecond)),
StartTime: types.NewTimeFromUnix(0, k.OpenTime*int64(time.Millisecond)),
EndTime: types.NewTimeFromUnix(0, k.CloseTime*int64(time.Millisecond)),
Open: util.MustParseFloat(k.Open),
Close: util.MustParseFloat(k.Close),
High: util.MustParseFloat(k.High),
@ -1070,7 +1070,7 @@ func (e *Exchange) BatchQueryKLines(ctx context.Context, symbol string, interval
}
allKLines = append(allKLines, kline)
startTime = kline.EndTime
startTime = kline.EndTime.Time()
}
}

4
pkg/exchange/binance/parse.go
View File

@ -521,8 +521,8 @@ func (k *KLine) KLine() types.KLine {
Exchange: types.ExchangeBinance,
Symbol: k.Symbol,
Interval: types.Interval(k.Interval),
StartTime: time.Unix(0, k.StartTime*int64(time.Millisecond)),
EndTime: time.Unix(0, k.EndTime*int64(time.Millisecond)),
StartTime: types.NewTimeFromUnix(0, k.StartTime*int64(time.Millisecond)),
EndTime: types.NewTimeFromUnix(0, k.EndTime*int64(time.Millisecond)),
Open: k.Open.Float64(),
Close: k.Close.Float64(),
High: k.High.Float64(),

4
pkg/exchange/binance/stream_test.go
View File

@ -49,8 +49,8 @@ func Test_Batch(t *testing.T) {
assert.True(t, nowMinTime.Unix() > lastmaxtime.Unix())
assert.True(t, nowMaxTime.Unix() > lastmaxtime.Unix())
lastmintime = nowMinTime
lastmaxtime = nowMaxTime
lastmintime = nowMinTime.Time()
lastmaxtime = nowMaxTime.Time()
assert.True(t, lastmintime.Unix() <= lastmaxtime.Unix())
}

4
pkg/exchange/ftx/convert.go
View File

@ -141,8 +141,8 @@ func toGlobalKLine(symbol string, interval types.Interval, h Candle) (types.KLin
return types.KLine{
Exchange: types.ExchangeFTX,
Symbol: toGlobalSymbol(symbol),
StartTime: h.StartTime.Time,
EndTime: h.StartTime.Add(interval.Duration()),
StartTime: types.Time(h.StartTime.Time),
EndTime: types.Time(h.StartTime.Add(interval.Duration())),
Interval: interval,
Open: h.Open,
Close: h.Close,

2
pkg/exchange/ftx/exchange.go
View File

@ -260,7 +260,7 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type
for _, line := range lines {
if line.StartTime.Unix() < currentEnd.Unix() {
currentEnd = line.StartTime
currentEnd = line.StartTime.Time()
}
if line.StartTime.Unix() > since.Unix() {

4
pkg/exchange/ftx/stream.go
View File

@ -145,7 +145,7 @@ func (s *Stream) pollKLines(ctx context.Context) {
s.EmitKLine(klines[0])
s.EmitKLineClosed(klines[0])
s.EmitKLine(klines[1])
lastClosed = klines[0].StartTime
lastClosed = klines[0].StartTime.Time()
}
}
@ -175,7 +175,7 @@ func (s *Stream) pollKLines(ctx context.Context) {
if lastClosed.Unix() < klines[0].StartTime.Unix() {
s.EmitKLine(klines[0])
s.EmitKLineClosed(klines[0])
lastClosed = klines[0].StartTime
lastClosed = klines[0].StartTime.Time()
}
s.EmitKLine(klines[1])
}

4
pkg/exchange/ftx/stream_test.go
View File

@ -80,8 +80,8 @@ func Test_Batch(t *testing.T) {
assert.True(t, nowMinTime.Unix() > lastmaxtime.Unix())
assert.True(t, nowMaxTime.Unix() > lastmaxtime.Unix())
}
lastmintime = nowMinTime
lastmaxtime = nowMaxTime
lastmintime = nowMinTime.Time()
lastmaxtime = nowMaxTime.Time()
assert.True(t, lastmintime.Unix() <= lastmaxtime.Unix())
}

4
pkg/exchange/max/maxapi/public.go
View File

@ -216,8 +216,8 @@ func (k KLine) KLine() types.KLine {
Exchange: types.ExchangeMax,
Symbol: strings.ToUpper(k.Symbol), // global symbol
Interval: types.Interval(k.Interval),
StartTime: k.StartTime,
EndTime: k.EndTime,
StartTime: types.Time(k.StartTime),
EndTime: types.Time(k.EndTime),
Open: k.Open,
Close: k.Close,
High: k.High,

4
pkg/exchange/max/maxapi/public_parser.go
View File

@ -118,8 +118,8 @@ type KLinePayload struct {
func (k KLinePayload) KLine() types.KLine {
return types.KLine{
StartTime: time.Unix(0, k.StartTime*int64(time.Millisecond)),
EndTime: time.Unix(0, k.EndTime*int64(time.Millisecond)),
StartTime: types.Time(time.Unix(0, k.StartTime*int64(time.Millisecond))),
EndTime: types.Time(time.Unix(0, k.EndTime*int64(time.Millisecond))),
Symbol: k.Market,
Interval: types.Interval(k.Resolution),
Open: util.MustParseFloat(k.Open),

4
pkg/exchange/okex/exchange.go
View File

@ -302,8 +302,8 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type
Closed: true,
Volume: candle.Volume.Float64(),
QuoteVolume: candle.VolumeInCurrency.Float64(),
StartTime: candle.Time,
EndTime: candle.Time.Add(interval.Duration() - time.Millisecond),
StartTime: types.Time(candle.Time),
EndTime: types.Time(candle.Time.Add(interval.Duration() - time.Millisecond)),
})
}

4
pkg/exchange/okex/parse.go
View File

@ -199,8 +199,8 @@ func (c *Candle) KLine() types.KLine {
Close: c.Close.Float64(),
Volume: c.Volume.Float64(),
QuoteVolume: c.VolumeInCurrency.Float64(),
StartTime: c.StartTime,
EndTime: endTime,
StartTime: types.Time(c.StartTime),
EndTime: types.Time(endTime),
}
}

2
pkg/indicator/ad.go
View File

@ -49,7 +49,7 @@ func (inc *AD) calculateAndUpdate(kLines []types.KLine) {
inc.update(k)
inc.EmitUpdate(inc.Last())
inc.EndTime = kLines[i].EndTime
inc.EndTime = kLines[i].EndTime.Time()
}
}

2
pkg/indicator/boll.go
View File

@ -105,7 +105,7 @@ func (inc *BOLL) calculateAndUpdate(kLines []types.KLine) {
inc.DownBand.Push(downBand)
// update end time
inc.EndTime = kLines[index].EndTime
inc.EndTime = kLines[index].EndTime.Time()
// log.Infof("update boll: sma=%f, up=%f, down=%f", sma, upBand, downBand)

2
pkg/indicator/ewma.go
View File

@ -81,7 +81,7 @@ func (inc *EWMA) calculateAndUpdate(allKLines []types.KLine) {
var k = allKLines[i]
var ewma = priceF(k)*multiplier + (1-multiplier)*inc.Values[i-1]
inc.Values.Push(ewma)
inc.LastOpenTime = k.StartTime
inc.LastOpenTime = k.StartTime.Time()
inc.EmitUpdate(ewma)
}

2
pkg/indicator/macd.go
View File

@ -76,7 +76,7 @@ func (inc *MACD) calculateAndUpdate(kLines []types.KLine) {
for i, kLine := range kLines {
inc.update(kLine, priceF)
inc.EmitUpdate(inc.Values[len(inc.Values)-1])
inc.EndTime = kLines[i].EndTime
inc.EndTime = kLines[i].EndTime.Time()
}
}

2
pkg/indicator/obv.go
View File

@ -59,7 +59,7 @@ func (inc *OBV) calculateAndUpdate(kLines []types.KLine) {
inc.update(k, priceF)
inc.EmitUpdate(inc.Last())
inc.EndTime = kLines[i].EndTime
inc.EndTime = kLines[i].EndTime.Time()
}
}

2
pkg/indicator/sma.go
View File

@ -55,7 +55,7 @@ func (inc *SMA) calculateAndUpdate(kLines []types.KLine) {
inc.Values = inc.Values[MaxNumOfSMATruncateSize-1:]
}
inc.EndTime = kLines[index].EndTime
inc.EndTime = kLines[index].EndTime.Time()
inc.EmitUpdate(sma)
}

2
pkg/indicator/stoch.go
View File

@ -66,7 +66,7 @@ func (inc *STOCH) calculateAndUpdate(kLines []types.KLine) {
inc.update(k)
inc.EmitUpdate(inc.LastK(), inc.LastD())
inc.EndTime = kLines[i].EndTime
inc.EndTime = kLines[i].EndTime.Time()
}
}

2
pkg/indicator/stoch_test.go
View File

@ -25,7 +25,7 @@ func TestSTOCH_update(t *testing.T) {
buildKLines := func(open, high, low, close []float64) (kLines []types.KLine) {
for i := range high {
kLines = append(kLines, types.KLine{Open: open[i], High: high[i], Low: low[i], Close: close[i], EndTime: time.Now()})
kLines = append(kLines, types.KLine{Open: open[i], High: high[i], Low: low[i], Close: close[i], EndTime: types.Time(time.Now())})
}
return kLines
}

2
pkg/indicator/vwap.go
View File

@ -92,7 +92,7 @@ func (inc *VWAP) calculateAndUpdate(kLines []types.KLine) {
inc.Values.Push(vwap)
inc.EmitUpdate(vwap)
inc.EndTime = kLines[i].EndTime
inc.EndTime = kLines[i].EndTime.Time()
}
}

6
pkg/service/backtest.go
View File

@ -65,7 +65,7 @@ func (s *BacktestService) Verify(symbols []string, startTime time.Time, endTime
if prevKLine.StartTime.Unix() == k.StartTime.Unix() {
s._deleteDuplicatedKLine(k)
log.Errorf("found kline data duplicated at time: %s kline: %+v , deleted it", k.StartTime, k)
} else if prevKLine.StartTime.Add(interval.Duration()) != k.StartTime {
} else if prevKLine.StartTime.Time().Add(interval.Duration()).Unix() != k.StartTime.Time().Unix() {
corruptCnt++
log.Errorf("found kline data corrupted at time: %s kline: %+v", k.StartTime, k)
log.Errorf("between %d and %d",
@ -339,9 +339,9 @@ func (s *BacktestService) SyncExist(ctx context.Context, exchange types.Exchange
for k := range klineC {
if nowStartTime.Add(interval.Duration()).Unix() < k.StartTime.Unix() {
log.Infof("syncing %s interval %s syncing %s ~ %s ", symbol, interval, nowStartTime, k.EndTime)
s.Sync(ctx, exchange, symbol, nowStartTime.Add(interval.Duration()), k.EndTime.Add(-1*interval.Duration()), interval)
s.Sync(ctx, exchange, symbol, nowStartTime.Add(interval.Duration()), k.EndTime.Time().Add(-1*interval.Duration()), interval)
}
nowStartTime = k.StartTime
nowStartTime = k.StartTime.Time()
}
if err := <-errC; err != nil {

11
pkg/types/kline.go
View File

@ -50,8 +50,8 @@ type KLine struct {
Symbol string `json:"symbol" db:"symbol"`
StartTime time.Time `json:"startTime" db:"start_time"`
EndTime time.Time `json:"endTime" db:"end_time"`
StartTime Time `json:"startTime" db:"start_time"`
EndTime Time `json:"endTime" db:"end_time"`
Interval Interval `json:"interval" db:"interval"`
@ -69,11 +69,11 @@ type KLine struct {
Closed bool `json:"closed" db:"closed"`
}
func (k KLine) GetStartTime() time.Time {
func (k KLine) GetStartTime() Time {
return k.StartTime
}
func (k KLine) GetEndTime() time.Time {
func (k KLine) GetEndTime() Time {
return k.EndTime
}
@ -179,11 +179,10 @@ func (k KLine) Color() string {
return GrayColor
}
func (k KLine) String() string {
return fmt.Sprintf("%s %s %s %s O: %.4f H: %.4f L: %.4f C: %.4f CHG: %.4f MAXCHG: %.4f V: %.4f QV: %.2f TBBV: %.2f",
k.Exchange.String(),
k.StartTime.Format("2006-01-02 15:04"),
k.StartTime.Time().Format("2006-01-02 15:04"),
k.Symbol, k.Interval, k.Open, k.High, k.Low, k.Close, k.GetChange(), k.GetMaxChange(), k.Volume, k.QuoteVolume, k.TakerBuyBaseAssetVolume)
}

16
pkg/types/time.go
View File

@ -96,6 +96,22 @@ func (t Time) Time() time.Time {
return time.Time(t)
}
func (t Time) Unix() int64 {
return time.Time(t).Unix()
}
func (t Time) After(time2 time.Time) bool {
return time.Time(t).After(time2)
}
func (t Time) Before(time2 time.Time) bool {
return time.Time(t).Before(time2)
}
func NewTimeFromUnix(sec int64, nsec int64) Time {
return Time(time.Unix(sec, nsec))
}
// Value implements the driver.Valuer interface
// see http://jmoiron.net/blog/built-in-interfaces/
func (t Time) Value() (driver.Value, error) {