lychiyu/bbgo_origin
1
0
Fork 0
mirror of https://github.com/c9s/bbgo.git synced 2024-09-20 08:11:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #117 from c9s/wizard/sqlite3

Browse Source 
add sqlite3 driver option to the wizard user interface
This commit is contained in:
Yo-An Lin 2021-02-06 17:41:45 +08:00 committed by GitHub
commit b81eb33cad
8 changed files with 228 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
README.md
View File

@ -56,35 +56,49 @@ MYSQL_URL=root@tcp(127.0.0.1:3306)/bbgo?parseTime=true
Make sure you have [dotenv](https://github.com/bkeepers/dotenv)
To sync your own trade data:
```
bbgo sync --config config/grid.yaml --session max
bbgo sync --config config/grid.yaml --session binance
```
If you want to switch to other dotenv file, you can add an `--dotenv` option:
```
bbgo sync --dotenv .env.dev --config config/grid.yaml --session binance
```
To sync remote exchange klines data for backtesting:
```sh
dotenv -f .env.local -- bbgo backtest --exchange binance --config config/grid.yaml -v --sync --sync-only --sync-from 2020-01-01
bbgo backtest --exchange binance --config config/grid.yaml -v --sync --sync-only --sync-from 2020-01-01
```
To run backtest:
```sh
dotenv -f .env.local -- bbgo backtest --exchange binance --config config/bollgrid.yaml --base-asset-baseline
bbgo backtest --exchange binance --config config/bollgrid.yaml --base-asset-baseline
```
To query transfer history:
```sh
dotenv -f .env.local -- bbgo transfer-history --exchange max --asset USDT --since "2019-01-01"
bbgo transfer-history --exchange max --asset USDT --since "2019-01-01"
```
To calculate pnl:
```sh
dotenv -f .env.local -- bbgo pnl --exchange binance --asset BTC --since "2019-01-01"
bbgo pnl --exchange binance --asset BTC --since "2019-01-01"
```
To run strategy:
```sh
dotenv -f .env.local -- bbgo run --config config/buyandhold.yaml
bbgo run --config config/buyandhold.yaml
```
## Built-in Strategies
@ -105,7 +119,7 @@ modify the config file to make the configuration suitable for you, for example i
vim config/buyandhold.yaml
# run bbgo with the config
dotenv -f .env.local -- bbgo run --config config/buyandhold.yaml
bbgo run --config config/buyandhold.yaml
```
## Write your own strategy

93
frontend/components/ConfigureDatabaseForm.js
View File

@ -5,10 +5,15 @@ import Button from '@material-ui/core/Button';
import Typography from '@material-ui/core/Typography';
import TextField from '@material-ui/core/TextField';
import FormHelperText from '@material-ui/core/FormHelperText';
import Radio from '@material-ui/core/Radio';
import RadioGroup from '@material-ui/core/RadioGroup';
import FormControlLabel from '@material-ui/core/FormControlLabel';
import FormControl from '@material-ui/core/FormControl';
import FormLabel from '@material-ui/core/FormLabel';
import Alert from '@material-ui/lab/Alert';
import {testDatabaseConnection, configureDatabase} from '../api/bbgo';
import {configureDatabase, testDatabaseConnection} from '../api/bbgo';
import {makeStyles} from '@material-ui/core/styles';
@ -30,10 +35,12 @@ const useStyles = makeStyles((theme) => ({
},
}));
export default function ConfigureDatabaseForm({ onConfigured }) {
export default function ConfigureDatabaseForm({onConfigured}) {
const classes = useStyles();
const [mysqlURL, setMysqlURL] = React.useState("root@tcp(127.0.0.1:3306)/bbgo")
const [driver, setDriver] = React.useState("sqlite3");
const [testing, setTesting] = React.useState(false);
const [testResponse, setTestResponse] = React.useState(null);
const [configured, setConfigured] = React.useState(false);
@ -79,44 +86,72 @@ export default function ConfigureDatabaseForm({ onConfigured }) {
</Typography>
<Typography variant="body1" gutterBottom>
If you have database installed on your machine, you can enter the DSN string in the following field.
Please note this is optional, you CAN SKIP this step.
If you have database installed on your machine, you can enter the DSN string in the following field.
Please note this is optional, you CAN SKIP this step.
</Typography>
<Grid container spacing={3}>
<Grid item xs={12} sm={6}>
<TextField id="mysql_url" name="mysql_url" label="MySQL Data Source Name"
fullWidth
required
defaultValue={mysqlURL}
onChange={(event) => {
setMysqlURL(event.target.value)
resetTestResponse()
}}
/>
<FormHelperText id="session-name-helper-text">
If you have database installed on your machine, you can enter the DSN string like the
<Grid container spacing={3}>
<Grid item xs={12} sm={4}>
<Box m={6}>
<FormControl component="fieldset" required={true}>
<FormLabel component="legend">Database Driver</FormLabel>
<RadioGroup aria-label="driver" name="driver" value={driver} onChange={(event) => {
setDriver(event.target.value);
}}>
<FormControlLabel value="sqlite3" control={<Radio/>} label="Standard (Default)"/>
<FormControlLabel value="mysql" control={<Radio/>} label="MySQL"/>
</RadioGroup>
</FormControl>
<FormHelperText>
</FormHelperText>
</Box>
</Grid>
{driver === "mysql" ? (
<Grid item xs={12} sm={8}>
<TextField id="mysql_url" name="mysql_url" label="MySQL Data Source Name"
fullWidth
required
defaultValue={mysqlURL}
onChange={(event) => {
setMysqlURL(event.target.value)
resetTestResponse()
}}
/>
<FormHelperText>MySQL DSN</FormHelperText>
<Typography variant="body1" gutterBottom>
If you have database installed on your machine, you can enter the DSN string like the
following
format:
<br/>
<code>
root:password@tcp(127.0.0.1:3306)/bbgo
</code>
<br/>
<pre><code>root:password@tcp(127.0.0.1:3306)/bbgo</code></pre>
<br/>
Be sure to create your database before using it. You need to execute the following statement
<br/>
Be sure to create your database before using it. You need to execute the following statement
to
create a database:
<br/>
<code>
CREATE DATABASE bbgo CHARSET utf8;
</code>
<br/>
<pre><code>CREATE DATABASE bbgo CHARSET utf8;</code></pre>
</Typography>
</FormHelperText>
</Grid>
</Grid>
) : (
<Grid item xs={12} sm={8}>
<Box m={6}>
<Typography variant="body1" gutterBottom>
If you don't know what to choose, just pick the standard driver (sqlite3).
<br/>
For professionals, you can pick MySQL driver, BBGO works best with MySQL, especially for
larger data scale.
</Typography>
</Box>
</Grid>
)}
</Grid>
<div className={classes.buttons}>
<Button
color="primary"

27
pkg/cmd/root.go
View File

@ -6,7 +6,9 @@ import (
"strings"
"time"
"github.com/joho/godotenv"
"github.com/lestrrat-go/file-rotatelogs"
"github.com/pkg/errors"
"github.com/rifflock/lfshook"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@ -23,6 +25,28 @@ var RootCmd = &cobra.Command{
// SilenceUsage is an option to silence usage when an error occurs.
SilenceUsage: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
disableDotEnv, err := cmd.Flags().GetBool("no-dotenv")
if err != nil {
return err
}
if !disableDotEnv {
dotenvFile, err := cmd.Flags().GetString("dotenv")
if err != nil {
return err
}
if _, err := os.Stat(dotenvFile); err == nil {
if err := godotenv.Load(dotenvFile); err != nil {
return errors.Wrap(err, "error loading dotenv file")
}
}
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
return nil
},
@ -32,6 +56,9 @@ func init() {
RootCmd.PersistentFlags().Bool("debug", false, "debug flag")
RootCmd.PersistentFlags().String("config", "bbgo.yaml", "config file")
RootCmd.PersistentFlags().Bool("no-dotenv", false, "disable built-in dotenv")
RootCmd.PersistentFlags().String("dotenv", ".env.local", "the dotenv file you want to load")
// A flag can be 'persistent' meaning that this flag will be available to
// the command it's assigned to as well as every command under that command.
// For global flags, assign a flag as a persistent flag on the root.

24
pkg/cmd/run.go
View File

@ -13,7 +13,6 @@ import (
"syscall"
"time"
"github.com/joho/godotenv"
"github.com/pkg/errors"
"github.com/pquerna/otp"
log "github.com/sirupsen/logrus"
@ -33,16 +32,11 @@ import (
func init() {
RunCmd.Flags().Bool("no-compile", false, "do not compile wrapper binary")
RunCmd.Flags().String("totp-key-url", "", "time-based one-time password key URL, if defined, it will be used for restoring the otp key")
RunCmd.Flags().String("totp-issuer", "", "")
RunCmd.Flags().String("totp-account-name", "", "")
RunCmd.Flags().Bool("enable-web-server", false, "enable web server")
RunCmd.Flags().Bool("setup", false, "use setup mode")
RunCmd.Flags().Bool("no-dotenv", false, "disable built-in dotenv")
RunCmd.Flags().String("dotenv", ".env.local", "the dotenv file you want to load")
RunCmd.Flags().String("since", "", "pnl since time")
RootCmd.AddCommand(RunCmd)
}
@ -306,24 +300,6 @@ func runConfig(basectx context.Context, userConfig *bbgo.Config, enableApiServer
}
func run(cmd *cobra.Command, args []string) error {
disableDotEnv, err := cmd.Flags().GetBool("no-dotenv")
if err != nil {
return err
}
if !disableDotEnv {
dotenvFile, err := cmd.Flags().GetString("dotenv")
if err != nil {
return err
}
if _, err := os.Stat(dotenvFile); err == nil {
if err := godotenv.Load(dotenvFile); err != nil {
return errors.Wrap(err, "error loading dotenv file")
}
}
}
setup, err := cmd.Flags().GetBool("setup")
if err != nil {
return err

44
pkg/server/routes.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io/ioutil"
"math/rand"
"net"
"net/http"
"os"
@ -17,6 +18,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/c9s/bbgo/pkg/bbgo"
"github.com/c9s/bbgo/pkg/fixedpoint"
"github.com/c9s/bbgo/pkg/service"
"github.com/c9s/bbgo/pkg/types"
)
@ -386,9 +388,46 @@ func (s *Server) listSessionOpenOrders(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"orders": marketOrders})
}
func (s *Server) listAssets(c *gin.Context) {
totalAssets := types.AssetMap{}
func genFakeAssets() types.AssetMap {
totalAssets := types.AssetMap{}
balances := types.BalanceMap{
"BTC": types.Balance{Currency: "BTC", Available: fixedpoint.NewFromFloat(10.0 * rand.Float64())},
"BCH": types.Balance{Currency: "BCH", Available: fixedpoint.NewFromFloat(0.01 * rand.Float64())},
"LTC": types.Balance{Currency: "LTC", Available: fixedpoint.NewFromFloat(200.0 * rand.Float64())},
"ETH": types.Balance{Currency: "ETH", Available: fixedpoint.NewFromFloat(50.0 * rand.Float64())},
"SAND": types.Balance{Currency: "SAND", Available: fixedpoint.NewFromFloat(11500.0 * rand.Float64())},
"BNB": types.Balance{Currency: "BNB", Available: fixedpoint.NewFromFloat(1000.0 * rand.Float64())},
"GRT": types.Balance{Currency: "GRT", Available: fixedpoint.NewFromFloat(1000.0 * rand.Float64())},
"MAX": types.Balance{Currency: "MAX", Available: fixedpoint.NewFromFloat(200000.0 * rand.Float64())},
"COMP": types.Balance{Currency: "COMP", Available: fixedpoint.NewFromFloat(100.0 * rand.Float64())},
}
assets := balances.Assets(map[string]float64{
"BTCUSDT": 38000.0,
"BCHUSDT": 478.0,
"LTCUSDT": 150.0,
"COMPUSDT": 450.0,
"ETHUSDT": 1700.0,
"BNBUSDT": 70.0,
"GRTUSDT": 0.89,
"DOTUSDT": 20.0,
"SANDUSDT": 0.13,
"MAXUSDT": 0.122,
})
for currency, asset := range assets {
totalAssets[currency] = asset
}
return totalAssets
}
func (s *Server) listAssets(c *gin.Context) {
if ok, err := strconv.ParseBool(os.Getenv("USE_FAKE_ASSETS")); err == nil && ok {
c.JSON(http.StatusOK, gin.H{"assets": genFakeAssets()})
return
}
totalAssets := types.AssetMap{}
for _, session := range s.Environ.Sessions() {
balances := session.Account.Balances()
@ -406,7 +445,6 @@ func (s *Server) listAssets(c *gin.Context) {
}
c.JSON(http.StatusOK, gin.H{"assets": totalAssets})
}
func (s *Server) setupSaveConfig(c *gin.Context) {

13
pkg/service/order.go
View File

@ -139,10 +139,19 @@ func (s *OrderService) scanRows(rows *sqlx.Rows) (orders []types.Order, err erro
return orders, rows.Err()
}
func (s *OrderService) Insert(order types.Order) error {
_, err := s.DB.NamedExec(`
func (s *OrderService) Insert(order types.Order) (err error) {
if s.DB.DriverName() == "mysql" {
_, err = s.DB.NamedExec(`
INSERT INTO orders (exchange, order_id, client_order_id, order_type, status, symbol, price, stop_price, quantity, executed_quantity, side, is_working, time_in_force, created_at, updated_at, is_margin, is_isolated)
VALUES (:exchange, :order_id, :client_order_id, :order_type, :status, :symbol, :price, :stop_price, :quantity, :executed_quantity, :side, :is_working, :time_in_force, :created_at, :updated_at, :is_margin, :is_isolated)
ON DUPLICATE KEY UPDATE status=:status, executed_quantity=:executed_quantity, is_working=:is_working, updated_at=:updated_at`, order)
return err
}
_, err = s.DB.NamedExec(`
INSERT INTO orders (exchange, order_id, client_order_id, order_type, status, symbol, price, stop_price, quantity, executed_quantity, side, is_working, time_in_force, created_at, updated_at, is_margin, is_isolated)
VALUES (:exchange, :order_id, :client_order_id, :order_type, :status, :symbol, :price, :stop_price, :quantity, :executed_quantity, :side, :is_working, :time_in_force, :created_at, :updated_at, :is_margin, :is_isolated)
`, order)
return err
}

66
pkg/service/trade.go
View File

@ -44,7 +44,15 @@ func (s *TradeService) QueryTradingVolume(startTime time.Time, options TradingVo
"start_time": startTime,
}
sql := queryTradingVolumeSQL(options)
sql := ""
driverName := s.DB.DriverName()
if driverName == "mysql" {
sql = generateMysqlTradingVolumeQuerySQL(options)
} else {
sql = generateSqliteTradingVolumeSQL(options)
}
log.Info(sql)
rows, err := s.DB.NamedQuery(sql, args)
if err != nil {
@ -72,14 +80,65 @@ func (s *TradeService) QueryTradingVolume(startTime time.Time, options TradingVo
return records, rows.Err()
}
func queryTradingVolumeSQL(options TradingVolumeQueryOptions) string {
func generateSqliteTradingVolumeSQL(options TradingVolumeQueryOptions) string {
var sel []string
var groupBys []string
var orderBys []string
where := []string{"traded_at > :start_time"}
switch options.GroupByPeriod {
switch options.GroupByPeriod {
case "month":
sel = append(sel, "strftime('%Y',traded_at) AS year", "strftime('%m',traded_at) AS month")
groupBys = append([]string{"month", "year"}, groupBys...)
orderBys = append(orderBys, "year ASC", "month ASC")
case "year":
sel = append(sel, "strftime('%Y',traded_at) AS year")
groupBys = append([]string{"year"}, groupBys...)
orderBys = append(orderBys, "year ASC")
case "day":
fallthrough
default:
sel = append(sel, "strftime('%Y',traded_at) AS year", "strftime('%m',traded_at) AS month", "strftime('%d',traded_at) AS day")
groupBys = append([]string{"day", "month", "year"}, groupBys...)
orderBys = append(orderBys, "year ASC", "month ASC", "day ASC")
}
switch options.SegmentBy {
case "symbol":
sel = append(sel, "symbol")
groupBys = append([]string{"symbol"}, groupBys...)
orderBys = append(orderBys, "symbol")
case "exchange":
sel = append(sel, "exchange")
groupBys = append([]string{"exchange"}, groupBys...)
orderBys = append(orderBys, "exchange")
}
sel = append(sel, "SUM(quantity * price) AS quote_volume")
sql := `SELECT ` + strings.Join(sel, ", ") + ` FROM trades` +
` WHERE ` + strings.Join(where, " AND ") +
` GROUP BY ` + strings.Join(groupBys, ", ") +
` ORDER BY ` + strings.Join(orderBys, ", ")
return sql
}
func generateMysqlTradingVolumeQuerySQL(options TradingVolumeQueryOptions) string {
var sel []string
var groupBys []string
var orderBys []string
where := []string{"traded_at > :start_time"}
switch options.GroupByPeriod {
case "month":
sel = append(sel, "YEAR(traded_at) AS year", "MONTH(traded_at) AS month")
groupBys = append([]string{"MONTH(traded_at)", "YEAR(traded_at)"}, groupBys...)
orderBys = append(orderBys, "year ASC", "month ASC")
@ -115,7 +174,6 @@ func queryTradingVolumeSQL(options TradingVolumeQueryOptions) string {
` GROUP BY ` + strings.Join(groupBys, ", ") +
` ORDER BY ` + strings.Join(orderBys, ", ")
log.Info(sql)
return sql
}

6
pkg/service/trade_test.go
View File

@ -11,15 +11,15 @@ func Test_queryTradingVolumeSQL(t *testing.T) {
o := TradingVolumeQueryOptions{
GroupByPeriod: "month",
}
assert.Equal(t, "SELECT YEAR(traded_at) AS year, MONTH(traded_at) AS month, SUM(quantity * price) AS quote_volume FROM trades WHERE traded_at > :start_time GROUP BY MONTH(traded_at), YEAR(traded_at) ORDER BY year ASC, month ASC", queryTradingVolumeSQL(o))
assert.Equal(t, "SELECT YEAR(traded_at) AS year, MONTH(traded_at) AS month, SUM(quantity * price) AS quote_volume FROM trades WHERE traded_at > :start_time GROUP BY MONTH(traded_at), YEAR(traded_at) ORDER BY year ASC, month ASC", generateMysqlTradingVolumeQuerySQL(o))
o.GroupByPeriod = "year"
assert.Equal(t, "SELECT YEAR(traded_at) AS year, SUM(quantity * price) AS quote_volume FROM trades WHERE traded_at > :start_time GROUP BY YEAR(traded_at) ORDER BY year ASC", queryTradingVolumeSQL(o))
assert.Equal(t, "SELECT YEAR(traded_at) AS year, SUM(quantity * price) AS quote_volume FROM trades WHERE traded_at > :start_time GROUP BY YEAR(traded_at) ORDER BY year ASC", generateMysqlTradingVolumeQuerySQL(o))
expectedDefaultSQL := "SELECT YEAR(traded_at) AS year, MONTH(traded_at) AS month, DAY(traded_at) AS day, SUM(quantity * price) AS quote_volume FROM trades WHERE traded_at > :start_time GROUP BY DAY(traded_at), MONTH(traded_at), YEAR(traded_at) ORDER BY year ASC, month ASC, day ASC"
for _, s := range []string{"", "day"} {
o.GroupByPeriod = s
assert.Equal(t, expectedDefaultSQL, queryTradingVolumeSQL(o))
assert.Equal(t, expectedDefaultSQL, generateMysqlTradingVolumeQuerySQL(o))
}
})