Support for systemd watchdog via sd_notify

This commit is contained in:
hroff-1902 2019-03-10 20:05:33 +03:00
parent e14739e102
commit 8730852d6e
9 changed files with 97 additions and 13 deletions

View File

@ -38,7 +38,7 @@ optional arguments:
--db-url PATH Override trades database URL, this is useful if --db-url PATH Override trades database URL, this is useful if
dry_run is enabled or in custom deployments (default: dry_run is enabled or in custom deployments (default:
None). None).
--sd-notify Notify systemd service manager.
``` ```
### How to use a different configuration file? ### How to use a different configuration file?

View File

@ -67,6 +67,7 @@ Mandatory Parameters are marked as **Required**.
| `strategy` | DefaultStrategy | Defines Strategy class to use. | `strategy` | DefaultStrategy | Defines Strategy class to use.
| `strategy_path` | null | Adds an additional strategy lookup path (must be a folder). | `strategy_path` | null | Adds an additional strategy lookup path (must be a folder).
| `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second.
| `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
### Parameters in the strategy ### Parameters in the strategy

View File

@ -428,6 +428,19 @@ For this to be persistent (run when user is logged out) you'll need to enable `l
sudo loginctl enable-linger "$USER" sudo loginctl enable-linger "$USER"
``` ```
If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot
state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the
configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd
using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped)
when it changes.
The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd
as the watchdog.
!!! Note:
The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a
Docker container.
------ ------
## Windows ## Windows

View File

@ -0,0 +1,30 @@
[Unit]
Description=Freqtrade Daemon
After=network.target
[Service]
# Set WorkingDirectory and ExecStart to your file paths accordingly
# NOTE: %h will be resolved to /home/<username>
WorkingDirectory=%h/freqtrade
ExecStart=/usr/bin/freqtrade --sd-notify
Restart=always
#Restart=on-failure
# Note that we use Type=notify here
Type=notify
# Currently required if Type=notify
NotifyAccess=all
StartLimitInterval=1min
StartLimitBurst=5
TimeoutStartSec=1min
# Use here (process_throttle_secs * 2) or longer time interval
WatchdogSec=20
[Install]
WantedBy=default.target

View File

@ -127,6 +127,12 @@ class Arguments(object):
type=str, type=str,
metavar='PATH', metavar='PATH',
) )
self.parser.add_argument(
'--sd-notify',
help='Notify systemd service manager.',
action='store_true',
dest='sd_notify',
)
@staticmethod @staticmethod
def backtesting_options(parser: argparse.ArgumentParser) -> None: def backtesting_options(parser: argparse.ArgumentParser) -> None:
@ -140,7 +146,6 @@ class Arguments(object):
dest='position_stacking', dest='position_stacking',
default=False default=False
) )
parser.add_argument( parser.add_argument(
'--dmmp', '--disable-max-market-positions', '--dmmp', '--disable-max-market-positions',
help='Disable applying `max_open_trades` during backtest ' help='Disable applying `max_open_trades` during backtest '

View File

@ -122,6 +122,10 @@ class Configuration(object):
set_loggers(config['verbosity']) set_loggers(config['verbosity'])
logger.info('Verbosity set to %s', config['verbosity']) logger.info('Verbosity set to %s', config['verbosity'])
# Support for sd_notify
if self.args.sd_notify:
config['internals'].update({'sd_notify': True})
# Add dynamic_whitelist if found # Add dynamic_whitelist if found
if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist:
# Update to volumePairList (the previous default) # Update to volumePairList (the previous default)

View File

@ -171,7 +171,8 @@ CONF_SCHEMA = {
'type': 'object', 'type': 'object',
'properties': { 'properties': {
'process_throttle_secs': {'type': 'number'}, 'process_throttle_secs': {'type': 'number'},
'interval': {'type': 'integer'} 'interval': {'type': 'integer'},
'sd_notify': {'type': 'boolean'},
} }
} }
}, },

View File

@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
import arrow import arrow
from requests.exceptions import RequestException from requests.exceptions import RequestException
import sdnotify
from freqtrade import (DependencyException, OperationalException, from freqtrade import (DependencyException, OperationalException,
TemporaryError, __version__, constants, persistence) TemporaryError, __version__, constants, persistence)
@ -51,6 +52,10 @@ class FreqtradeBot(object):
# Init objects # Init objects
self.config = config self.config = config
self._sd_notify = sdnotify.SystemdNotifier() if \
self.config.get('internals', {}).get('sd_notify', False) else None
self.strategy: IStrategy = StrategyResolver(self.config).strategy self.strategy: IStrategy = StrategyResolver(self.config).strategy
self.rpc: RPCManager = RPCManager(self) self.rpc: RPCManager = RPCManager(self)
@ -76,6 +81,11 @@ class FreqtradeBot(object):
self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist']
self._init_modules() self._init_modules()
# Tell the systemd that we completed initialization phase
if self._sd_notify:
logger.debug("sd_notify: READY=1")
self._sd_notify.notify('READY=1')
def _init_modules(self) -> None: def _init_modules(self) -> None:
""" """
Initializes all modules and updates the config Initializes all modules and updates the config
@ -99,6 +109,12 @@ class FreqtradeBot(object):
:return: None :return: None
""" """
logger.info('Cleaning up modules ...') logger.info('Cleaning up modules ...')
# Tell systemd that we are stopping now
if self._sd_notify:
logger.debug("sd_notify: STOPPING=1")
self._sd_notify.notify('STOPPING=1')
self.rpc.cleanup() self.rpc.cleanup()
persistence.cleanup() persistence.cleanup()
@ -119,16 +135,27 @@ class FreqtradeBot(object):
if state == State.RUNNING: if state == State.RUNNING:
self.rpc.startup_messages(self.config, self.pairlists) self.rpc.startup_messages(self.config, self.pairlists)
if state == State.STOPPED: throttle_secs = self.config.get('internals', {}).get(
time.sleep(1)
elif state == State.RUNNING:
min_secs = self.config.get('internals', {}).get(
'process_throttle_secs', 'process_throttle_secs',
constants.PROCESS_THROTTLE_SECS constants.PROCESS_THROTTLE_SECS
) )
self._throttle(func=self._process, if state == State.STOPPED:
min_secs=min_secs) # Ping systemd watchdog before sleeping in the stopped state
if self._sd_notify:
logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.")
self._sd_notify.notify('WATCHDOG=1\nSTATUS=State: STOPPED.')
time.sleep(throttle_secs)
elif state == State.RUNNING:
# Ping systemd watchdog before throttling
if self._sd_notify:
logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.")
self._sd_notify.notify('WATCHDOG=1\nSTATUS=State: RUNNING.')
self._throttle(func=self._process, min_secs=throttle_secs)
return state return state
def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any:

View File

@ -24,3 +24,6 @@ py_find_1st==1.1.3
# Load ticker files 30% faster # Load ticker files 30% faster
python-rapidjson==0.7.0 python-rapidjson==0.7.0
# Notify systemd
sdnotify==0.3.2