bbgo_origin/pkg/cache/cache.go

179 lines
4.0 KiB
Go
Raw Normal View History

package cache
2020-10-20 04:11:44 +00:00
import (
2021-02-19 02:42:24 +00:00
"context"
2020-10-20 04:11:44 +00:00
"encoding/json"
2021-02-19 02:42:24 +00:00
"fmt"
2020-10-20 04:11:44 +00:00
"io/ioutil"
"os"
"path"
"reflect"
2023-04-14 07:23:34 +00:00
"sync"
2022-01-06 15:57:42 +00:00
"time"
2020-10-20 04:11:44 +00:00
2021-12-08 09:26:25 +00:00
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
2022-01-06 15:57:42 +00:00
"github.com/c9s/bbgo/pkg/types"
2023-04-14 07:23:34 +00:00
"github.com/c9s/bbgo/pkg/util"
"github.com/c9s/bbgo/pkg/util/backoff"
2020-10-20 04:11:44 +00:00
)
2023-04-14 07:23:34 +00:00
const memCacheExpiry = 5 * time.Minute
const fileCacheExpiry = 24 * time.Hour
var globalMarketMemCache *marketMemCache = newMarketMemCache()
type marketMemCache struct {
sync.Mutex
markets map[string]marketMapWithTime
}
type marketMapWithTime struct {
updatedAt time.Time
markets types.MarketMap
}
func newMarketMemCache() *marketMemCache {
cache := &marketMemCache{
markets: make(map[string]marketMapWithTime),
}
return cache
}
func (c *marketMemCache) IsOutdated(exName string) bool {
c.Lock()
defer c.Unlock()
data, ok := c.markets[exName]
return !ok || time.Since(data.updatedAt) > memCacheExpiry
}
func (c *marketMemCache) Set(exName string, markets types.MarketMap) {
c.Lock()
defer c.Unlock()
c.markets[exName] = marketMapWithTime{
updatedAt: time.Now(),
markets: markets,
}
}
func (c *marketMemCache) Get(exName string) (types.MarketMap, bool) {
c.Lock()
defer c.Unlock()
markets, ok := c.markets[exName]
if !ok {
return nil, false
}
2020-10-20 04:22:18 +00:00
2023-04-14 07:23:34 +00:00
copied := types.MarketMap{}
for key, val := range markets.markets {
copied[key] = val
}
return copied, true
}
type DataFetcher func() (interface{}, error)
2022-01-06 15:57:42 +00:00
2020-10-20 04:22:18 +00:00
// WithCache let you use the cache with the given cache key, variable reference and your data fetcher,
// The key must be an unique ID.
// obj is the pointer of your local variable
// fetcher is the closure that will fetch your remote data or some slow operation.
func WithCache(key string, obj interface{}, fetcher DataFetcher) error {
2020-10-20 04:11:44 +00:00
cacheDir := CacheDir()
2020-10-20 04:24:30 +00:00
cacheFile := path.Join(cacheDir, key+".json")
2020-10-20 04:11:44 +00:00
2022-01-06 15:57:42 +00:00
stat, err := os.Stat(cacheFile)
2023-04-14 07:23:34 +00:00
if os.IsNotExist(err) || (stat != nil && time.Since(stat.ModTime()) > fileCacheExpiry) {
2022-01-06 15:57:42 +00:00
log.Debugf("cache %s not found or cache expired, executing fetcher callback to get the data", cacheFile)
2020-10-20 04:22:18 +00:00
data, err := fetcher()
2020-10-20 04:11:44 +00:00
if err != nil {
return err
}
out, err := json.Marshal(data)
if err != nil {
return err
}
if err := ioutil.WriteFile(cacheFile, out, 0666); err != nil {
return err
}
rv := reflect.ValueOf(obj).Elem()
if !rv.CanSet() {
return errors.New("can not set cache object value")
}
rv.Set(reflect.ValueOf(data))
} else {
log.Debugf("cache %s found", cacheFile)
2020-10-20 04:11:44 +00:00
data, err := ioutil.ReadFile(cacheFile)
if err != nil {
return err
}
if err := json.Unmarshal(data, obj); err != nil {
return err
}
}
return nil
}
2021-02-19 02:42:24 +00:00
2024-03-12 03:10:42 +00:00
func LoadExchangeMarketsWithCache(ctx context.Context, ex types.ExchangePublic) (markets types.MarketMap, err error) {
2023-04-14 07:23:34 +00:00
inMem, ok := util.GetEnvVarBool("USE_MARKETS_CACHE_IN_MEMORY")
if ok && inMem {
return loadMarketsFromMem(ctx, ex)
}
// fallback to use files as cache
return loadMarketsFromFile(ctx, ex)
}
// loadMarketsFromMem is useful for one process to run multiple bbgos in different go routines.
2024-03-12 03:10:42 +00:00
func loadMarketsFromMem(ctx context.Context, ex types.ExchangePublic) (markets types.MarketMap, _ error) {
2023-04-14 07:23:34 +00:00
exName := ex.Name().String()
if globalMarketMemCache.IsOutdated(exName) {
op := func() error {
rst, err2 := ex.QueryMarkets(ctx)
if err2 != nil {
return err2
}
markets = rst
globalMarketMemCache.Set(exName, rst)
return nil
}
if err := backoff.RetryGeneral(ctx, op); err != nil {
return nil, err
}
return markets, nil
}
rst, _ := globalMarketMemCache.Get(exName)
return rst, nil
}
2024-03-12 03:10:42 +00:00
func loadMarketsFromFile(ctx context.Context, ex types.ExchangePublic) (markets types.MarketMap, err error) {
key := fmt.Sprintf("%s-markets", ex.Name())
2021-12-08 09:26:25 +00:00
if futureExchange, implemented := ex.(types.FuturesExchange); implemented {
settings := futureExchange.GetFuturesSettings()
if settings.IsFutures {
key = fmt.Sprintf("%s-futures-markets", ex.Name())
}
}
err = WithCache(key, &markets, func() (interface{}, error) {
2021-02-19 02:42:24 +00:00
return ex.QueryMarkets(ctx)
})
return markets, err
2023-05-25 06:01:22 +00:00
}