Merge pull request #601 from c9s/feature/backtest-report

apps/bbgo-backtest-report/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

apps/bbgo-backtest-report/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo

apps/bbgo-backtest-report/README.md

@@ -0,0 +1,34 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file.
+
+[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
+
+The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

apps/bbgo-backtest-report/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/basic-features/typescript for more information.

apps/bbgo-backtest-report/next.config.js

@@ -0,0 +1,6 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  reactStrictMode: true,
+}
+
+module.exports = nextConfig

apps/bbgo-backtest-report/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "bbgo-backtest-report",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "d3-dsv": "^3.0.1",
+    "d3-format": "^3.1.0",
+    "d3-time-format": "^4.1.0",
+    "klinecharts": "^8.3.6",
+    "next": "12.1.6",
+    "react": "18.1.0",
+    "react-dom": "18.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "17.0.31",
+    "@types/react": "18.0.8",
+    "@types/react-dom": "18.0.3",
+    "eslint": "8.14.0",
+    "eslint-config-next": "12.1.6",
+    "typescript": "4.6.4"
+  }
+}

apps/bbgo-backtest-report/pages/_app.tsx

@@ -0,0 +1,8 @@
+import '../styles/globals.css'
+import type { AppProps } from 'next/app'
+
+function MyApp({ Component, pageProps }: AppProps) {
+  return <Component {...pageProps} />
+}
+
+export default MyApp

apps/bbgo-backtest-report/pages/api/hello.ts

@@ -0,0 +1,13 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next'
+
+type Data = {
+  name: string
+}
+
+export default function handler(
+  req: NextApiRequest,
+  res: NextApiResponse<Data>
+) {
+  res.status(200).json({ name: 'John Doe' })
+}

apps/bbgo-backtest-report/pages/index.tsx

@@ -0,0 +1,72 @@
+import type { NextPage } from 'next'
+import Head from 'next/head'
+import Image from 'next/image'
+import styles from '../styles/Home.module.css'
+
+const Home: NextPage = () => {
+  return (
+    <div className={styles.container}>
+      <Head>
+        <title>Create Next App</title>
+        <meta name="description" content="Generated by create next app" />
+        <link rel="icon" href="/favicon.ico" />
+      </Head>
+
+      <main className={styles.main}>
+        <h1 className={styles.title}>
+          Welcome to <a href="https://nextjs.org">Next.js!</a>
+        </h1>
+
+        <p className={styles.description}>
+          Get started by editing{' '}
+          <code className={styles.code}>pages/index.tsx</code>
+        </p>
+
+        <div className={styles.grid}>
+          <a href="https://nextjs.org/docs" className={styles.card}>
+            <h2>Documentation &rarr;</h2>
+            <p>Find in-depth information about Next.js features and API.</p>
+          </a>
+
+          <a href="https://nextjs.org/learn" className={styles.card}>
+            <h2>Learn &rarr;</h2>
+            <p>Learn about Next.js in an interactive course with quizzes!</p>
+          </a>
+
+          <a
+            href="https://github.com/vercel/next.js/tree/canary/examples"
+            className={styles.card}
+          >
+            <h2>Examples &rarr;</h2>
+            <p>Discover and deploy boilerplate example Next.js projects.</p>
+          </a>
+
+          <a
+            href="https://vercel.com/new?utm_source=create-next-app&utm_medium=default-template&utm_campaign=create-next-app"
+            className={styles.card}
+          >
+            <h2>Deploy &rarr;</h2>
+            <p>
+              Instantly deploy your Next.js site to a public URL with Vercel.
+            </p>
+          </a>
+        </div>
+      </main>
+
+      <footer className={styles.footer}>
+        <a
+          href="https://vercel.com?utm_source=create-next-app&utm_medium=default-template&utm_campaign=create-next-app"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          Powered by{' '}
+          <span className={styles.logo}>
+            <Image src="/vercel.svg" alt="Vercel Logo" width={72} height={16} />
+          </span>
+        </a>
+      </footer>
+    </div>
+  )
+}
+
+export default Home

apps/bbgo-backtest-report/public/vercel.svg

@@ -0,0 +1,4 @@
+<svg width="283" height="64" viewBox="0 0 283 64" fill="none" 
+    xmlns="http://www.w3.org/2000/svg">
+    <path d="M141.04 16c-11.04 0-19 7.2-19 18s8.96 18 20 18c6.67 0 12.55-2.64 16.19-7.09l-7.65-4.42c-2.02 2.21-5.09 3.5-8.54 3.5-4.79 0-8.86-2.5-10.37-6.5h28.02c.22-1.12.35-2.28.35-3.5 0-10.79-7.96-17.99-19-17.99zm-9.46 14.5c1.25-3.99 4.67-6.5 9.45-6.5 4.79 0 8.21 2.51 9.45 6.5h-18.9zM248.72 16c-11.04 0-19 7.2-19 18s8.96 18 20 18c6.67 0 12.55-2.64 16.19-7.09l-7.65-4.42c-2.02 2.21-5.09 3.5-8.54 3.5-4.79 0-8.86-2.5-10.37-6.5h28.02c.22-1.12.35-2.28.35-3.5 0-10.79-7.96-17.99-19-17.99zm-9.45 14.5c1.25-3.99 4.67-6.5 9.45-6.5 4.79 0 8.21 2.51 9.45 6.5h-18.9zM200.24 34c0 6 3.92 10 10 10 4.12 0 7.21-1.87 8.8-4.92l7.68 4.43c-3.18 5.3-9.14 8.49-16.48 8.49-11.05 0-19-7.2-19-18s7.96-18 19-18c7.34 0 13.29 3.19 16.48 8.49l-7.68 4.43c-1.59-3.05-4.68-4.92-8.8-4.92-6.07 0-10 4-10 10zm82.48-29v46h-9V5h9zM36.95 0L73.9 64H0L36.95 0zm92.38 5l-27.71 48L73.91 5H84.3l17.32 30 17.32-30h10.39zm58.91 12v9.69c-1-.29-2.06-.49-3.2-.49-5.81 0-10 4-10 10V51h-9V17h9v9.2c0-5.08 5.91-9.2 13.2-9.2z" fill="#000"/>
+</svg>

apps/bbgo-backtest-report/src/Chart.js

@@ -0,0 +1,131 @@
+
+import { format } from "d3-format";
+import { timeFormat } from "d3-time-format";
+
+import { ChartCanvas, Chart } from "react-stockcharts";
+import {
+    CandlestickSeries,
+    LineSeries,
+} from "react-stockcharts/lib/series";
+import { XAxis, YAxis } from "react-stockcharts/lib/axes";
+import {
+    CrossHairCursor,
+    EdgeIndicator,
+    CurrentCoordinate,
+    MouseCoordinateX,
+    MouseCoordinateY,
+} from "react-stockcharts/lib/coordinates";
+
+import { LabelAnnotation, Label, Annotate } from "react-stockcharts/lib/annotation";
+import { discontinuousTimeScaleProvider } from "react-stockcharts/lib/scale";
+import { OHLCTooltip, MovingAverageTooltip } from "react-stockcharts/lib/tooltip";
+import { ema } from "react-stockcharts/lib/indicator";
+// import { fitWidth } from "react-stockcharts/lib/helper";
+import { last } from "react-stockcharts/lib/utils";
+
+let CandleStickChartWithAnnotation = function(props) {
+        const annotationProps = {
+            fontFamily: "Glyphicons Halflings",
+            fontSize: 20,
+            fill: "#060F8F",
+            opacity: 0.8,
+            text: "\ue182",
+            y: ({ yScale }) => yScale.range()[0],
+            onClick: console.log.bind(console),
+            tooltip: d => timeFormat("%B")(d.date),
+            // onMouseOver: console.log.bind(console),
+        };
+
+        const margin = { left: 80, right: 80, top: 30, bottom: 50 };
+        const height = 400;
+        const { type, data: initialData, width, ratio } = props;
+
+        const [yAxisLabelX, yAxisLabelY] = [
+            width - margin.left - 40,
+            (height - margin.top - margin.bottom) / 2
+        ];
+
+        const xScaleProvider = discontinuousTimeScaleProvider
+            .inputDateAccessor(d => d.date);
+        const {
+            data,
+            xScale,
+            xAccessor,
+            displayXAccessor,
+        } = xScaleProvider(initialData);
+
+        const start = xAccessor(last(data));
+        const end = xAccessor(data[Math.max(0, data.length - 150)]);
+        const xExtents = [start, end];
+
+        return (
+            <ChartCanvas height={height}
+                         ratio={ratio}
+                         width={width}
+                         margin={margin}
+                         type={type}
+                         seriesName="MSFT"
+                         data={data}
+                         xScale={xScale}
+                         xAccessor={xAccessor}
+                         displayXAccessor={displayXAccessor}
+                         xExtents={xExtents}>
+
+                <Label x={(width - margin.left - margin.right) / 2} y={30}
+                       fontSize="30" text="Chart title here" />
+
+                <Chart id={1}
+                       yExtents={[d => [d.high, d.low]]}
+                       padding={{ top: 10, bottom: 20 }}>
+                    <XAxis axisAt="bottom" orient="bottom"/>
+                    <MouseCoordinateX
+                        at="bottom"
+                        orient="bottom"
+                        displayFormat={timeFormat("%Y-%m-%d")} />
+                    <MouseCoordinateY
+                        at="right"
+                        orient="right"
+                        displayFormat={format(".2f")} />
+
+                    <Label x={(width - margin.left - margin.right) / 2} y={height - 45}
+                           fontSize="12" text="XAxis Label here" />
+
+                    <YAxis axisAt="right" orient="right" ticks={5} />
+
+                    <Label x={yAxisLabelX} y={yAxisLabelY}
+                           rotate={-90}
+                           fontSize="12" text="YAxis Label here" />
+
+                    <CandlestickSeries />
+                    <EdgeIndicator itemType="last" orient="right" edgeAt="right"
+                                   yAccessor={d => d.close} fill={d => d.close > d.open ? "#6BA583" : "#FF0000"}/>
+
+                    <OHLCTooltip origin={[-40, 0]}/>
+
+                    <Annotate with={LabelAnnotation}
+                              when={d => d.date.getDate() === 1 /* some condition */}
+                              usingProps={annotationProps} />
+
+                </Chart>
+                <CrossHairCursor strokeDasharray="LongDashDot" />
+
+            </ChartCanvas>
+        );
+}
+
+/*
+CandleStickChartWithAnnotation.propTypes = {
+    data: PropTypes.array.isRequired,
+    width: PropTypes.number.isRequired,
+    ratio: PropTypes.number.isRequired,
+    type: PropTypes.oneOf(["svg", "hybrid"]).isRequired,
+};
+
+CandleStickChartWithAnnotation.defaultProps = {
+    type: "svg",
+};
+*/
+
+// CandleStickChartWithAnnotation = fitWidth(CandleStickChartWithAnnotation);
+
+export default CandleStickChartWithAnnotation;

apps/bbgo-backtest-report/src/utils.js

@@ -0,0 +1,25 @@
+import { tsvParse, csvParse } from  "d3-dsv";
+import { timeParse } from "d3-time-format";
+
+function parseData(parse) {
+    return function(d) {
+        d.date = parse(d.date);
+        d.open = +d.open;
+        d.high = +d.high;
+        d.low = +d.low;
+        d.close = +d.close;
+        d.volume = +d.volume;
+
+        return d;
+    };
+}
+
+const parseDate = timeParse("%Y-%m-%d");
+
+export function getData() {
+    // original source: https://cdn.rawgit.com/rrag/react-stockcharts/master/docs/data/MSFT.tsv
+    const promiseMSFT = fetch("https://cdn.jsdelivr.net/gh/rrag/react-stockcharts@master/docs/data/MSFT.tsv")
+        .then(response => response.text())
+        .then(data => tsvParse(data, parseData(parseDate)))
+    return promiseMSFT;
+}

apps/bbgo-backtest-report/styles/Home.module.css

@@ -0,0 +1,116 @@
+.container {
+  padding: 0 2rem;
+}
+
+.main {
+  min-height: 100vh;
+  padding: 4rem 0;
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+}
+
+.footer {
+  display: flex;
+  flex: 1;
+  padding: 2rem 0;
+  border-top: 1px solid #eaeaea;
+  justify-content: center;
+  align-items: center;
+}
+
+.footer a {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  flex-grow: 1;
+}
+
+.title a {
+  color: #0070f3;
+  text-decoration: none;
+}
+
+.title a:hover,
+.title a:focus,
+.title a:active {
+  text-decoration: underline;
+}
+
+.title {
+  margin: 0;
+  line-height: 1.15;
+  font-size: 4rem;
+}
+
+.title,
+.description {
+  text-align: center;
+}
+
+.description {
+  margin: 4rem 0;
+  line-height: 1.5;
+  font-size: 1.5rem;
+}
+
+.code {
+  background: #fafafa;
+  border-radius: 5px;
+  padding: 0.75rem;
+  font-size: 1.1rem;
+  font-family: Menlo, Monaco, Lucida Console, Liberation Mono, DejaVu Sans Mono,
+    Bitstream Vera Sans Mono, Courier New, monospace;
+}
+
+.grid {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  flex-wrap: wrap;
+  max-width: 800px;
+}
+
+.card {
+  margin: 1rem;
+  padding: 1.5rem;
+  text-align: left;
+  color: inherit;
+  text-decoration: none;
+  border: 1px solid #eaeaea;
+  border-radius: 10px;
+  transition: color 0.15s ease, border-color 0.15s ease;
+  max-width: 300px;
+}
+
+.card:hover,
+.card:focus,
+.card:active {
+  color: #0070f3;
+  border-color: #0070f3;
+}
+
+.card h2 {
+  margin: 0 0 1rem 0;
+  font-size: 1.5rem;
+}
+
+.card p {
+  margin: 0;
+  font-size: 1.25rem;
+  line-height: 1.5;
+}
+
+.logo {
+  height: 1em;
+  margin-left: 0.5rem;
+}
+
+@media (max-width: 600px) {
+  .grid {
+    width: 100%;
+    flex-direction: column;
+  }
+}

apps/bbgo-backtest-report/styles/globals.css

@@ -0,0 +1,16 @@
+html,
+body {
+  padding: 0;
+  margin: 0;
+  font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen,
+    Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif;
+}
+
+a {
+  color: inherit;
+  text-decoration: none;
+}
+
+* {
+  box-sizing: border-box;
+}

apps/bbgo-backtest-report/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+  "exclude": ["node_modules"]
+}

pkg/backtest/dumper.go

@@ -0,0 +1,108 @@
+package backtest
+
+import (
+	"encoding/csv"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"go.uber.org/multierr"
+
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+const DateFormat = "2006-01-02T15:04"
+
+type symbolInterval struct {
+	Symbol   string
+	Interval types.Interval
+}
+
+// KLineDumper dumps the received kline data into a folder for the backtest report to load the charts.
+type KLineDumper struct {
+	OutputDirectory string
+	files           map[symbolInterval]*os.File
+	writers         map[symbolInterval]*csv.Writer
+	filenames       map[symbolInterval]string
+}
+
+func NewKLineDumper(outputDirectory string) *KLineDumper {
+	return &KLineDumper{
+		OutputDirectory: outputDirectory,
+		files:           make(map[symbolInterval]*os.File),
+		writers:         make(map[symbolInterval]*csv.Writer),
+		filenames:       make(map[symbolInterval]string),
+	}
+}
+
+func (d *KLineDumper) Filenames() map[symbolInterval]string {
+	return d.filenames
+}
+
+func (d *KLineDumper) formatFileName(symbol string, interval types.Interval) string {
+	return filepath.Join(d.OutputDirectory, fmt.Sprintf("%s-%s.csv",
+		symbol,
+		interval))
+}
+
+var csvHeader = []string{"date", "startTime", "endTime", "interval", "open", "high", "low", "close", "volume"}
+
+func (d *KLineDumper) encode(k types.KLine) []string {
+	return []string{
+		time.Time(k.StartTime).Format(time.ANSIC), // ANSIC date - for javascript to parse (this works with Date.parse(date_str)
+		strconv.FormatInt(k.StartTime.Unix(), 10),
+		strconv.FormatInt(k.EndTime.Unix(), 10),
+		k.Interval.String(),
+		k.Open.String(),
+		k.High.String(),
+		k.Low.String(),
+		k.Close.String(),
+		k.Volume.String(),
+	}
+}
+
+func (d *KLineDumper) Record(k types.KLine) error {
+	si := symbolInterval{Symbol: k.Symbol, Interval: k.Interval}
+
+	w, ok := d.writers[si]
+	if !ok {
+		filename := d.formatFileName(k.Symbol, k.Interval)
+		f2, err := os.Create(filename)
+		if err != nil {
+			return err
+		}
+
+		w = csv.NewWriter(f2)
+		d.files[si] = f2
+		d.writers[si] = w
+		d.filenames[si] = filename
+
+		if err := w.Write(csvHeader); err != nil {
+			return err
+		}
+	}
+
+	if err := w.Write(d.encode(k)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *KLineDumper) Close() error {
+	for _, w := range d.writers {
+		w.Flush()
+	}
+
+	var err error = nil
+	for _, f := range d.files {
+		err2 := f.Close()
+		if err2 != nil {
+			err = multierr.Append(err, err2)
+		}
+	}
+
+	return err
+}

pkg/backtest/dumper_test.go

@@ -0,0 +1,54 @@
+package backtest
+
+import (
+	"encoding/csv"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+func TestKLineDumper(t *testing.T) {
+	tempDir := os.TempDir()
+	_ = os.Mkdir(tempDir, 0755)
+	dumper := NewKLineDumper(tempDir)
+
+	t1 := time.Now()
+	err := dumper.Record(types.KLine{
+		Exchange:                 types.ExchangeBinance,
+		Symbol:                   "BTCUSDT",
+		StartTime:                types.Time(t1),
+		EndTime:                  types.Time(t1.Add(time.Minute)),
+		Interval:                 types.Interval1m,
+		Open:                     fixedpoint.NewFromFloat(1000.0),
+		High:                     fixedpoint.NewFromFloat(2000.0),
+		Low:                      fixedpoint.NewFromFloat(3000.0),
+		Close:                    fixedpoint.NewFromFloat(4000.0),
+		Volume:                   fixedpoint.NewFromFloat(5000.0),
+		QuoteVolume:              fixedpoint.NewFromFloat(6000.0),
+		NumberOfTrades:           10,
+		Closed:                   true,
+	})
+	assert.NoError(t, err)
+
+	err = dumper.Close()
+	assert.NoError(t, err)
+
+	filenames := dumper.Filenames()
+	assert.NotEmpty(t, filenames)
+	for _, filename := range filenames {
+		f, err := os.Open(filename)
+		if assert.NoError(t, err) {
+			reader := csv.NewReader(f)
+			records, err2 := reader.Read()
+			if assert.NoError(t, err2) {
+				assert.NotEmptyf(t, records, "%v", records)
+			}
+
+		}
+	}
+}

pkg/backtest/exchange_klinec.go

@@ -0,0 +1,8 @@
+package backtest
+
+import "github.com/c9s/bbgo/pkg/types"
+
+type ExchangeDataSource struct {
+	C        chan types.KLine
+	Exchange *Exchange
+}

pkg/backtest/report.go

@@ -0,0 +1,32 @@
+package backtest
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/c9s/bbgo/pkg/accounting/pnl"
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+type Report struct {
+	Symbol          string                    `json:"symbol,omitempty"`
+	LastPrice       fixedpoint.Value          `json:"lastPrice,omitempty"`
+	StartPrice      fixedpoint.Value          `json:"startPrice,omitempty"`
+	PnLReport       *pnl.AverageCostPnlReport `json:"pnlReport,omitempty"`
+	InitialBalances types.BalanceMap          `json:"initialBalances,omitempty"`
+	FinalBalances   types.BalanceMap          `json:"finalBalances,omitempty"`
+}
+
+const SessionTimeFormat = "2006-01-02T15_04"
+
+// FormatSessionName returns the back-test session name
+func FormatSessionName(sessions []string, symbols []string, startTime, endTime time.Time) string {
+	return fmt.Sprintf("%s_%s_%s-%s",
+		strings.Join(sessions, "-"),
+		strings.Join(symbols, "-"),
+		startTime.Format(SessionTimeFormat),
+		endTime.Format(SessionTimeFormat),
+	)
+}

pkg/bbgo/environment.go

@@ -239,6 +239,10 @@ func (environ *Environment) AddExchangesFromSessionConfig(sessions map[string]*E
 	return nil
 }
 
+func (environ *Environment) IsBackTesting() bool {
+	return environ.BacktestService != nil
+}
+
 // Init prepares the data that will be used by the strategies
 func (environ *Environment) Init(ctx context.Context) (err error) {
 	for n := range environ.sessions {

pkg/cmd/backtest.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/fatih/color"
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -22,25 +23,17 @@ import (
 	"github.com/c9s/bbgo/pkg/backtest"
 	"github.com/c9s/bbgo/pkg/bbgo"
 	"github.com/c9s/bbgo/pkg/cmd/cmdutil"
-	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/service"
 	"github.com/c9s/bbgo/pkg/types"
 )
 
-type BackTestReport struct {
-	Symbol          string                    `json:"symbol,omitempty"`
-	LastPrice       fixedpoint.Value          `json:"lastPrice,omitempty"`
-	StartPrice      fixedpoint.Value          `json:"startPrice,omitempty"`
-	PnLReport       *pnl.AverageCostPnlReport `json:"pnlReport,omitempty"`
-	InitialBalances types.BalanceMap          `json:"initialBalances,omitempty"`
-	FinalBalances   types.BalanceMap          `json:"finalBalances,omitempty"`
-}
-
 func init() {
 	BacktestCmd.Flags().Bool("sync", false, "sync backtest data")
 	BacktestCmd.Flags().Bool("sync-only", false, "sync backtest data only, do not run backtest")
 	BacktestCmd.Flags().String("sync-from", "", "sync backtest data from the given time, which will override the time range in the backtest config")
 	BacktestCmd.Flags().String("sync-exchange", "", "specify only one exchange to sync backtest data")
+	BacktestCmd.Flags().String("session", "", "specify only one exchange session to run backtest")
+
 	BacktestCmd.Flags().Bool("verify", false, "verify the kline back-test data")
 
 	BacktestCmd.Flags().Bool("base-asset-baseline", false, "use base asset performance as the competitive baseline performance")
@@ -48,12 +41,13 @@ func init() {
 	BacktestCmd.Flags().String("config", "config/bbgo.yaml", "strategy config file")
 	BacktestCmd.Flags().Bool("force", false, "force execution without confirm")
 	BacktestCmd.Flags().String("output", "", "the report output directory")
+	BacktestCmd.Flags().Bool("subdir", false, "generate report in the sub-directory of the output directory")
 	RootCmd.AddCommand(BacktestCmd)
 }
 
 var BacktestCmd = &cobra.Command{
 	Use:          "backtest",
-	Short:        "backtest your strategies",
+	Short:        "run backtest with strategies",
 	SilenceUsage: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		verboseCnt, err := cmd.Flags().GetCount("verbose")
@@ -89,6 +83,11 @@ var BacktestCmd = &cobra.Command{
 			return err
 		}
 
+		sessionName, err := cmd.Flags().GetString("session")
+		if err != nil {
+			return err
+		}
+
 		force, err := cmd.Flags().GetBool("force")
 		if err != nil {
 			return err
@@ -99,6 +98,13 @@ var BacktestCmd = &cobra.Command{
 			return err
 		}
 
+		generatingReport := len(outputDirectory) > 0
+
+		reportFileInSubDir, err := cmd.Flags().GetBool("subdir")
+		if err != nil {
+			return err
+		}
+
 		jsonOutputEnabled := len(outputDirectory) > 0
 
 		syncOnly, err := cmd.Flags().GetBool("sync-only")
@@ -166,9 +172,20 @@ var BacktestCmd = &cobra.Command{
 		backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
 		environ.BacktestService = backtestService
 
+		if len(sessionName) > 0 {
+			userConfig.Backtest.Sessions = []string{sessionName}
+		} else if len(syncExchangeName) > 0 {
+			userConfig.Backtest.Sessions = []string{syncExchangeName}
+		} else if len(userConfig.Backtest.Sessions) == 0 {
+			log.Infof("backtest.sessions is not defined, using all supported exchanges: %v", types.SupportedExchanges)
+			for _, exName := range types.SupportedExchanges {
+				userConfig.Backtest.Sessions = append(userConfig.Backtest.Sessions, exName.String())
+			}
+		}
+
 		var sourceExchanges = make(map[types.ExchangeName]types.Exchange)
-		if len(syncExchangeName) > 0 {
-			exName, err := types.ValidExchangeName(syncExchangeName)
+		for _, name := range userConfig.Backtest.Sessions {
+			exName, err := types.ValidExchangeName(name)
 			if err != nil {
 				return err
 			}
@@ -178,29 +195,6 @@ var BacktestCmd = &cobra.Command{
 				return err
 			}
 			sourceExchanges[exName] = publicExchange
-
-		} else if len(userConfig.Backtest.Sessions) > 0 {
-			for _, name := range userConfig.Backtest.Sessions {
-				exName, err := types.ValidExchangeName(name)
-				if err != nil {
-					return err
-				}
-
-				publicExchange, err := cmdutil.NewExchangePublic(exName)
-				if err != nil {
-					return err
-				}
-				sourceExchanges[exName] = publicExchange
-			}
-		} else {
-			log.Infof("backtest.sessions is not defined, loading all supported exchanges: %v", types.SupportedExchanges)
-			for _, exName := range types.SupportedExchanges {
-				publicExchange, err := cmdutil.NewExchangePublic(exName)
-				if err != nil {
-					return err
-				}
-				sourceExchanges[exName] = publicExchange
-			}
 		}
 
 		if wantSync {
@@ -317,37 +311,97 @@ var BacktestCmd = &cobra.Command{
 			return err
 		}
 
-		type KChanEx struct {
-			KChan    chan types.KLine
-			Exchange *backtest.Exchange
-		}
 		for _, session := range environ.Sessions() {
 			backtestExchange := session.Exchange.(*backtest.Exchange)
 			backtestExchange.InitMarketData()
 		}
 
-		var klineChans []KChanEx
+		var exchangeSources []backtest.ExchangeDataSource
 		for _, session := range environ.Sessions() {
 			exchange := session.Exchange.(*backtest.Exchange)
 			c, err := exchange.GetMarketData()
 			if err != nil {
 				return err
 			}
-			klineChans = append(klineChans, KChanEx{KChan: c, Exchange: exchange})
+			exchangeSources = append(exchangeSources, backtest.ExchangeDataSource{C: c, Exchange: exchange})
+		}
+
+		// back-test session report name
+		var backtestSessionName = backtest.FormatSessionName(
+			userConfig.Backtest.Sessions,
+			userConfig.Backtest.Symbols,
+			userConfig.Backtest.StartTime.Time(),
+			userConfig.Backtest.EndTime.Time(),
+		)
+
+		var kLineHandlers []func(k types.KLine)
+		if generatingReport {
+			dumpDir := outputDirectory
+			if reportFileInSubDir {
+				dumpDir = filepath.Join(dumpDir, backtestSessionName)
+				dumpDir = filepath.Join(dumpDir, uuid.NewString())
+			}
+
+			dumpDir = filepath.Join(dumpDir, "klines")
+
+			if _, err := os.Stat(dumpDir); err != nil {
+				if os.IsNotExist(err) {
+					if err2 := os.MkdirAll(dumpDir, 0755); err2 != nil {
+						return err2
+					}
+				} else {
+					return err
+				}
+			}
+
+			dumper := backtest.NewKLineDumper(dumpDir)
+			defer func() {
+				if err := dumper.Close(); err != nil {
+					log.WithError(err).Errorf("kline dumper can not close files")
+				}
+			}()
+
+			kLineHandlers = append(kLineHandlers, func(k types.KLine) {
+				if err := dumper.Record(k); err != nil {
+					log.WithError(err).Errorf("can not write kline to file")
+				}
+			})
 		}
 
 		runCtx, cancelRun := context.WithCancel(ctx)
 		go func() {
 			defer cancelRun()
+
+			// Optimize back-test speed for single exchange source
+			var count = len(exchangeSources)
+			if count == 1 {
+				exSource := exchangeSources[0]
+				for k := range exSource.C {
+					exSource.Exchange.ConsumeKLine(k)
+
+					for _, h := range kLineHandlers {
+						h(k)
+					}
+				}
+
+				if err := exSource.Exchange.CloseMarketData(); err != nil {
+					log.WithError(err).Errorf("close market data error")
+				}
+				return
+			}
+
 			for {
-				count := len(klineChans)
-				for _, kchanex := range klineChans {
-					kLine, more := <-kchanex.KChan
+				for _, exK := range exchangeSources {
+					k, more := <-exK.C
 					if more {
-						kchanex.Exchange.ConsumeKLine(kLine)
+						exK.Exchange.ConsumeKLine(k)
+
+						for _, h := range kLineHandlers {
+							h(k)
+						}
 					} else {
-						if err := kchanex.Exchange.CloseMarketData(); err != nil {
-							log.Errorf("%v", err)
+						if err := exK.Exchange.CloseMarketData(); err != nil {
+							log.WithError(err).Errorf("close market data error")
 							return
 						}
 						count--
@@ -397,7 +451,7 @@ var BacktestCmd = &cobra.Command{
 
 				report := calculator.Calculate(symbol, trades.Trades, lastPrice)
 				report.Print()
-				
+
 				accountConfig, ok := userConfig.Backtest.Accounts[exchangeName]
 				if !ok {
 					accountConfig = userConfig.Backtest.Account[exchangeName]
@@ -413,7 +467,7 @@ var BacktestCmd = &cobra.Command{
 				finalBalances.Print()
 
 				if jsonOutputEnabled {
-					result := BackTestReport{
+					result := backtest.Report{
 						Symbol:          symbol,
 						LastPrice:       lastPrice,
 						StartPrice:      startPrice,

pkg/strategy/bollmaker/strategy.go

@@ -631,6 +631,11 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se
 
 	s.SmartStops.RunStopControllers(ctx, session, s.tradeCollector)
 
+	if s.Environment.IsBackTesting() {
+		log.Warn("turning of useTickerPrice option in the back-testing environment...")
+		s.UseTickerPrice = false
+	}
+
 	session.UserDataStream.OnStart(func() {
 		if s.UseTickerPrice {
 			ticker, err := s.session.Exchange.QueryTicker(ctx, s.Symbol)