mirror of
https://github.com/c9s/bbgo.git
synced 2024-11-23 23:35:14 +00:00
281 lines
6.7 KiB
Go
281 lines
6.7 KiB
Go
package csvsource
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
log "github.com/sirupsen/logrus"
|
|
|
|
"github.com/c9s/bbgo/pkg/exchange/kucoin"
|
|
"github.com/c9s/bbgo/pkg/types"
|
|
)
|
|
|
|
type MarketType string
|
|
type DataType string
|
|
|
|
const (
|
|
SPOT MarketType = "spot"
|
|
FUTURES MarketType = "futures"
|
|
TRADES DataType = "trades"
|
|
AGGTRADES DataType = "aggTrades"
|
|
// todo could be extended to:
|
|
|
|
// LEVEL2 = 2
|
|
// https://data.binance.vision/data/futures/um/daily/bookTicker/ADAUSDT/ADAUSDT-bookTicker-2023-11-19.zip
|
|
// update_id best_bid_price best_bid_qty best_ask_price best_ask_qty transaction_time event_time
|
|
// 3.52214E+12 0.3772 1632 0.3773 67521 1.70035E+12 1.70035E+12
|
|
|
|
// METRICS = 3
|
|
// https://data.binance.vision/data/futures/um/daily/metrics/ADAUSDT/ADAUSDT-metrics-2023-11-19.zip
|
|
// create_time symbol sum_open_interest sum_open_interest_value count_toptrader_long_short_ratio sum_toptrader_long_short_ratio count_long_short_ratio sum_taker_long_short_vol_ratio
|
|
// 19/11/2023 00:00 ADAUSDT 141979878.00000000 53563193.89339590 2.33412322 1.21401178 2.46604727 0.55265805
|
|
|
|
// KLINES DataType = 4
|
|
// https://public.bybit.com/kline_for_metatrader4/BNBUSDT/2021/BNBUSDT_15_2021-07-01_2021-07-31.csv.gz
|
|
// only few symbols but supported interval options 1m/ 5m/ 15m/ 30m/ 60m/ and only monthly
|
|
|
|
// https://data.binance.vision/data/futures/um/daily/klines/1INCHBTC/30m/1INCHBTC-30m-2023-11-18.zip
|
|
// supported interval options 1s/ 1m/ 3m/ 5m/ 15m/ 30m/ 1h/ 2h/ 4h/ 6h/ 8h/ 12h/ 1d/ daily or monthly futures
|
|
|
|
// this might be useful for backtesting against mark or index price
|
|
// especially index price can be used across exchanges
|
|
// https://data.binance.vision/data/futures/um/daily/indexPriceKlines/ADAUSDT/1h/ADAUSDT-1h-2023-11-19.zip
|
|
// https://data.binance.vision/data/futures/um/daily/markPriceKlines/ADAUSDT/1h/ADAUSDT-1h-2023-11-19.zip
|
|
|
|
// OKex or Bybit do not support direct kLine, metrics or level2 csv download
|
|
|
|
)
|
|
|
|
func Download(
|
|
path, symbol string,
|
|
exchange types.ExchangeName,
|
|
market MarketType,
|
|
granularity DataType,
|
|
since, until time.Time,
|
|
) (err error) {
|
|
for {
|
|
var (
|
|
fileName = fmt.Sprintf("%s-%s.csv", symbol, since.Format("2006-01-02"))
|
|
)
|
|
|
|
if fileExists(filepath.Join(path, fileName)) {
|
|
since = since.AddDate(0, 0, 1)
|
|
continue
|
|
}
|
|
|
|
var url, err = buildURL(exchange, symbol, market, granularity, fileName, since)
|
|
if err != nil {
|
|
log.Error(err)
|
|
break
|
|
}
|
|
|
|
log.Info("fetching ", url)
|
|
|
|
csvContent, err := readCSVFromUrl(exchange, url)
|
|
if err != nil {
|
|
log.Error(err)
|
|
break
|
|
}
|
|
|
|
err = write(csvContent, fmt.Sprintf("%s/%s", path, granularity), fileName)
|
|
if err != nil {
|
|
log.Error(err)
|
|
break
|
|
}
|
|
|
|
since = since.AddDate(0, 0, 1)
|
|
if since.After(until) {
|
|
break
|
|
}
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func buildURL(
|
|
exchange types.ExchangeName,
|
|
symbol string,
|
|
market MarketType,
|
|
granularity DataType,
|
|
fileName string,
|
|
start time.Time,
|
|
) (url string, err error) {
|
|
switch exchange {
|
|
case types.ExchangeBybit:
|
|
// bybit doesn't seem to differentiate between spot and futures market or trade type in their csv dumps ;(
|
|
url = fmt.Sprintf("https://public.bybit.com/trading/%s/%s%s.csv.gz",
|
|
symbol,
|
|
symbol,
|
|
start.Format("2006-01-02"),
|
|
)
|
|
case types.ExchangeBinance:
|
|
marketType := "spot"
|
|
if market == FUTURES {
|
|
marketType = "futures/um"
|
|
}
|
|
dataType := "aggTrades"
|
|
if granularity == TRADES {
|
|
dataType = "trades"
|
|
}
|
|
url = fmt.Sprintf("https://data.binance.vision/data/%s/daily/%s/%s/%s-%s-%s.zip",
|
|
marketType,
|
|
dataType,
|
|
symbol,
|
|
symbol,
|
|
dataType,
|
|
start.Format("2006-01-02"))
|
|
|
|
case types.ExchangeOKEx:
|
|
// todo temporary find a better solution ?!
|
|
coins := strings.Split(kucoin.ToLocalSymbol(symbol), "-")
|
|
if len(coins) == 0 {
|
|
err = fmt.Errorf("%s not supported yet for OKEx.. care to fix it? PR's welcome ;)", symbol)
|
|
return
|
|
}
|
|
baseCoin := coins[0]
|
|
quoteCoin := coins[1]
|
|
marketType := "" // for spot market
|
|
if market == FUTURES {
|
|
marketType = "-SWAP"
|
|
}
|
|
dataType := "aggtrades"
|
|
if granularity == TRADES {
|
|
dataType = "trades"
|
|
}
|
|
url = fmt.Sprintf("https://static.okx.com/cdn/okex/traderecords/%s/daily/%s/%s-%s%s-%s-%s.zip",
|
|
dataType,
|
|
start.Format("20060102"),
|
|
baseCoin,
|
|
quoteCoin,
|
|
marketType,
|
|
dataType,
|
|
start.Format("2006-01-02"))
|
|
default:
|
|
err = fmt.Errorf("%s not supported yet as csv data source.. care to fix it? PR's welcome ;)", exchange.String())
|
|
}
|
|
|
|
return url, err
|
|
}
|
|
|
|
func readCSVFromUrl(exchange types.ExchangeName, url string) (csvContent []byte, err error) {
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("http get error, url %s: %w", url, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to read response: %w", err)
|
|
}
|
|
|
|
switch exchange {
|
|
case types.ExchangeBybit:
|
|
csvContent, err = gunzip(body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("gunzip data %s: %w", exchange, err)
|
|
}
|
|
|
|
case types.ExchangeBinance:
|
|
csvContent, err = unzip(body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unzip data %s: %w", exchange, err)
|
|
}
|
|
|
|
case types.ExchangeOKEx:
|
|
csvContent, err = unzip(body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unzip data %s: %w", exchange, err)
|
|
}
|
|
}
|
|
|
|
return csvContent, nil
|
|
}
|
|
|
|
func write(content []byte, path, fileName string) error {
|
|
|
|
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
|
|
err := os.MkdirAll(path, os.ModePerm)
|
|
if err != nil {
|
|
return fmt.Errorf("mkdir %s: %w", path, err)
|
|
}
|
|
}
|
|
|
|
dest := filepath.Join(path, fileName)
|
|
|
|
err := os.WriteFile(dest, content, 0666)
|
|
if err != nil {
|
|
return fmt.Errorf("write %s: %w", dest, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func unzip(data []byte) (resData []byte, err error) {
|
|
zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
|
if err != nil {
|
|
log.Error(err)
|
|
}
|
|
|
|
if zipReader == nil || len(zipReader.File) == 0 {
|
|
return nil, errors.New("no data to unzip")
|
|
}
|
|
|
|
// Read all the files from zip archive
|
|
for _, zipFile := range zipReader.File {
|
|
resData, err = readZipFile(zipFile)
|
|
if err != nil {
|
|
log.Error(err)
|
|
break
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func readZipFile(zf *zip.File) ([]byte, error) {
|
|
f, err := zf.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer f.Close()
|
|
return io.ReadAll(f)
|
|
}
|
|
|
|
func gunzip(data []byte) (resData []byte, err error) {
|
|
b := bytes.NewBuffer(data)
|
|
|
|
var r io.Reader
|
|
r, err = gzip.NewReader(b)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
var resB bytes.Buffer
|
|
_, err = resB.ReadFrom(r)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
resData = resB.Bytes()
|
|
|
|
return
|
|
}
|
|
|
|
func fileExists(fileName string) bool {
|
|
info, err := os.Stat(fileName)
|
|
if os.IsNotExist(err) {
|
|
return false
|
|
}
|
|
return !info.IsDir()
|
|
}
|