diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 92e60daa4..f3f56c7b2 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -16,6 +16,7 @@ from freqtrade.exceptions import ConfigurationError from freqtrade.exchange import timeframe_to_minutes from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.resolvers import ExchangeResolver +from freqtrade.util import print_rich_table from freqtrade.util.migrations import migrate_data @@ -119,8 +120,6 @@ def start_list_data(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - from tabulate import tabulate - from freqtrade.data.history import get_datahandler dhc = get_datahandler(config["datadir"], config["dataformat_ohlcv"]) @@ -131,8 +130,7 @@ def start_list_data(args: Dict[str, Any]) -> None: if args["pairs"]: paircombs = [comb for comb in paircombs if comb[0] in args["pairs"]] - - print(f"Found {len(paircombs)} pair / timeframe combinations.") + title = f"Found {len(paircombs)} pair / timeframe combinations." if not config.get("show_timerange"): groupedpair = defaultdict(list) for pair, timeframe, candle_type in sorted( @@ -141,40 +139,35 @@ def start_list_data(args: Dict[str, Any]) -> None: groupedpair[(pair, candle_type)].append(timeframe) if groupedpair: - print( - tabulate( - [ - (pair, ", ".join(timeframes), candle_type) - for (pair, candle_type), timeframes in groupedpair.items() - ], - headers=("Pair", "Timeframe", "Type"), - tablefmt="psql", - stralign="right", - ) + print_rich_table( + [ + (pair, ", ".join(timeframes), candle_type) + for (pair, candle_type), timeframes in groupedpair.items() + ], + ("Pair", "Timeframe", "Type"), + title, + table_kwargs={"min_width": 50}, ) else: paircombs1 = [ (pair, timeframe, candle_type, *dhc.ohlcv_data_min_max(pair, timeframe, candle_type)) for pair, timeframe, candle_type in paircombs ] - - print( - tabulate( - [ - ( - pair, - timeframe, - candle_type, - start.strftime(DATETIME_PRINT_FORMAT), - end.strftime(DATETIME_PRINT_FORMAT), - length, - ) - for pair, timeframe, candle_type, start, end, length in sorted( - paircombs1, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]) - ) - ], - headers=("Pair", "Timeframe", "Type", "From", "To", "Candles"), - tablefmt="psql", - stralign="right", - ) + print_rich_table( + [ + ( + pair, + timeframe, + candle_type, + start.strftime(DATETIME_PRINT_FORMAT), + end.strftime(DATETIME_PRINT_FORMAT), + str(length), + ) + for pair, timeframe, candle_type, start, end, length in sorted( + paircombs1, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]) + ) + ], + ("Pair", "Timeframe", "Type", "From", "To", "Candles"), + summary=title, + table_kwargs={"min_width": 50}, ) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index ac0b8453f..d89d25796 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -2,8 +2,6 @@ import logging from operator import itemgetter from typing import Any, Dict -from colorama import init as colorama_init - from freqtrade.configuration import setup_utils_configuration from freqtrade.data.btanalysis import get_latest_hyperopt_file from freqtrade.enums import RunMode @@ -18,6 +16,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: """ List hyperopt epochs previously evaluated """ + from freqtrade.optimize.hyperopt_output import HyperoptOutput from freqtrade.optimize.hyperopt_tools import HyperoptTools config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -35,21 +34,17 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: # Previous evaluations epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config) - if print_colorized: - colorama_init(autoreset=True) - if not export_csv: try: - print( - HyperoptTools.get_result_table( - config, - epochs, - total_epochs, - not config.get("hyperopt_list_best", False), - print_colorized, - 0, - ) + h_out = HyperoptOutput() + h_out.add_data( + config, + epochs, + total_epochs, + not config.get("hyperopt_list_best", False), ) + h_out.print(print_colorized=print_colorized) + except KeyboardInterrupt: print("User interrupted..") diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 97be7bac6..1696fc8f0 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -4,9 +4,9 @@ import sys from typing import Any, Dict, List, Union import rapidjson -from colorama import Fore, Style -from colorama import init as colorama_init -from tabulate import tabulate +from rich.console import Console +from rich.table import Table +from rich.text import Text from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode @@ -14,7 +14,8 @@ from freqtrade.exceptions import ConfigurationError, OperationalException from freqtrade.exchange import list_available_exchanges, market_is_active from freqtrade.misc import parse_db_uri_for_logging, plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.types import ValidExchangesType +from freqtrade.types.valid_exchanges_type import ValidExchangesType +from freqtrade.util import print_rich_table logger = logging.getLogger(__name__) @@ -26,72 +27,69 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ - exchanges = list_available_exchanges(args["list_exchanges_all"]) + available_exchanges: List[ValidExchangesType] = list_available_exchanges( + args["list_exchanges_all"] + ) if args["print_one_column"]: - print("\n".join([e["name"] for e in exchanges])) + print("\n".join([e["name"] for e in available_exchanges])) else: - headers = { - "name": "Exchange name", - "supported": "Supported", - "trade_modes": "Markets", - "comment": "Reason", - } - headers.update({"valid": "Valid"} if args["list_exchanges_all"] else {}) + if args["list_exchanges_all"]: + title = ( + f"All exchanges supported by the ccxt library " + f"({len(available_exchanges)} exchanges):" + ) + else: + available_exchanges = [e for e in available_exchanges if e["valid"] is not False] + title = f"Exchanges available for Freqtrade ({len(available_exchanges)} exchanges):" - def build_entry(exchange: ValidExchangesType, valid: bool): - valid_entry = {"valid": exchange["valid"]} if valid else {} - result: Dict[str, Union[str, bool]] = { - "name": exchange["name"], - **valid_entry, - "supported": "Official" if exchange["supported"] else "", - "trade_modes": ("DEX: " if exchange["dex"] else "") - + ", ".join( - (f"{a['margin_mode']} " if a["margin_mode"] else "") + a["trading_mode"] + table = Table(title=title) + + table.add_column("Exchange Name") + table.add_column("Markets") + table.add_column("Reason") + + for exchange in available_exchanges: + name = Text(exchange["name"]) + if exchange["supported"]: + name.append(" (Official)", style="italic") + name.stylize("green bold") + + trade_modes = Text( + ", ".join( + (f"{a.get('margin_mode', '')} {a['trading_mode']}").lstrip() for a in exchange["trade_modes"] ), - "comment": exchange["comment"], - } - - return result - - if args["list_exchanges_all"]: - exchanges = [build_entry(e, True) for e in exchanges] - print(f"All exchanges supported by the ccxt library ({len(exchanges)} exchanges):") - else: - exchanges = [build_entry(e, False) for e in exchanges if e["valid"] is not False] - print(f"Exchanges available for Freqtrade ({len(exchanges)} exchanges):") - - print( - tabulate( - exchanges, - headers=headers, + style="", ) - ) + if exchange["dex"]: + trade_modes = Text("DEX: ") + trade_modes + trade_modes.stylize("bold", 0, 3) + + table.add_row( + name, + trade_modes, + exchange["comment"], + style=None if exchange["valid"] else "red", + ) + # table.add_row(*[exchange[header] for header in headers]) + + console = Console() + console.print(table) def _print_objs_tabular(objs: List, print_colorized: bool) -> None: - if print_colorized: - colorama_init(autoreset=True) - red = Fore.RED - yellow = Fore.YELLOW - reset = Style.RESET_ALL - else: - red = "" - yellow = "" - reset = "" - names = [s["name"] for s in objs] - objs_to_print = [ + objs_to_print: List[Dict[str, Union[Text, str]]] = [ { - "name": s["name"] if s["name"] else "--", + "name": Text(s["name"] if s["name"] else "--"), "location": s["location_rel"], "status": ( - red + "LOAD FAILED" + reset + Text("LOAD FAILED", style="bold red") if s["class"] is None - else "OK" + else Text("OK", style="bold green") if names.count(s["name"]) == 1 - else yellow + "DUPLICATE NAME" + reset + else Text("DUPLICATE NAME", style="bold yellow") ), } for s in objs @@ -101,11 +99,23 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: objs_to_print[idx].update( { "hyperoptable": "Yes" if s["hyperoptable"]["count"] > 0 else "No", - "buy-Params": len(s["hyperoptable"].get("buy", [])), - "sell-Params": len(s["hyperoptable"].get("sell", [])), + "buy-Params": str(len(s["hyperoptable"].get("buy", []))), + "sell-Params": str(len(s["hyperoptable"].get("sell", []))), } ) - print(tabulate(objs_to_print, headers="keys", tablefmt="psql", stralign="right")) + table = Table() + + for header in objs_to_print[0].keys(): + table.add_column(header.capitalize(), justify="right") + + for row in objs_to_print: + table.add_row(*[row[header] for header in objs_to_print[0].keys()]) + + console = Console( + color_system="auto" if print_colorized else None, + width=200 if "pytest" in sys.modules else None, + ) + console.print(table) def start_list_strategies(args: Dict[str, Any]) -> None: @@ -270,9 +280,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: writer.writeheader() writer.writerows(tabular_data) else: - # print data as a table, with the human-readable summary - print(f"{summary_str}:") - print(tabulate(tabular_data, headers="keys", tablefmt="psql", stralign="right")) + print_rich_table(tabular_data, headers, summary_str) elif not ( args.get("print_one_column", False) or args.get("list_pairs_print_json", False) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 9d936d295..e76f2dff7 100644 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -4,7 +4,6 @@ from typing import List import joblib import pandas as pd -from tabulate import tabulate from freqtrade.configuration import TimeRange from freqtrade.constants import Config @@ -14,6 +13,7 @@ from freqtrade.data.btanalysis import ( load_backtest_stats, ) from freqtrade.exceptions import OperationalException +from freqtrade.util import print_df_rich_table logger = logging.getLogger(__name__) @@ -307,7 +307,7 @@ def _print_table( if name is not None: print(name) - print(tabulate(data, headers="keys", tablefmt="psql", showindex=show_index)) + print_df_rich_table(data, data.keys(), show_index=show_index) def process_entry_exit_reasons(config: Config): diff --git a/freqtrade/optimize/analysis/lookahead_helpers.py b/freqtrade/optimize/analysis/lookahead_helpers.py index c0e6fa1ba..a8fb1cd35 100644 --- a/freqtrade/optimize/analysis/lookahead_helpers.py +++ b/freqtrade/optimize/analysis/lookahead_helpers.py @@ -4,11 +4,13 @@ from pathlib import Path from typing import Any, Dict, List import pandas as pd +from rich.text import Text from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.optimize.analysis.lookahead import LookaheadAnalysis from freqtrade.resolvers import StrategyResolver +from freqtrade.util import print_rich_table logger = logging.getLogger(__name__) @@ -53,18 +55,18 @@ class LookaheadAnalysisSubFunctions: [ inst.strategy_obj["location"].parts[-1], inst.strategy_obj["name"], - inst.current_analysis.has_bias, + Text("Yes", style="bold red") + if inst.current_analysis.has_bias + else Text("No", style="bold green"), inst.current_analysis.total_signals, inst.current_analysis.false_entry_signals, inst.current_analysis.false_exit_signals, ", ".join(inst.current_analysis.false_indicators), ] ) - from tabulate import tabulate - table = tabulate(data, headers=headers, tablefmt="orgtbl") - print(table) - return table, headers, data + print_rich_table(data, headers, summary="Lookahead Analysis") + return data @staticmethod def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]): diff --git a/freqtrade/optimize/analysis/recursive_helpers.py b/freqtrade/optimize/analysis/recursive_helpers.py index cde1a214e..be596fa68 100644 --- a/freqtrade/optimize/analysis/recursive_helpers.py +++ b/freqtrade/optimize/analysis/recursive_helpers.py @@ -7,6 +7,7 @@ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.optimize.analysis.recursive import RecursiveAnalysis from freqtrade.resolvers import StrategyResolver +from freqtrade.util import print_rich_table logger = logging.getLogger(__name__) @@ -16,9 +17,9 @@ class RecursiveAnalysisSubFunctions: @staticmethod def text_table_recursive_analysis_instances(recursive_instances: List[RecursiveAnalysis]): startups = recursive_instances[0]._startup_candle - headers = ["indicators"] + headers = ["Indicators"] for candle in startups: - headers.append(candle) + headers.append(str(candle)) data = [] for inst in recursive_instances: @@ -30,13 +31,11 @@ class RecursiveAnalysisSubFunctions: data.append(temp_data) if len(data) > 0: - from tabulate import tabulate + print_rich_table(data, headers, summary="Recursive Analysis") - table = tabulate(data, headers=headers, tablefmt="orgtbl") - print(table) - return table, headers, data + return data - return None, None, data + return data @staticmethod def calculate_config_overrides(config: Config): diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b19fca9dc..b411e7752 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,14 +14,14 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import rapidjson -from colorama import init as colorama_init from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects from joblib.externals import cloudpickle from pandas import DataFrame +from rich.align import Align +from rich.console import Console from rich.progress import ( BarColumn, MofNCompleteColumn, - Progress, TaskProgressColumn, TextColumn, TimeElapsedColumn, @@ -40,6 +40,7 @@ from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss +from freqtrade.optimize.hyperopt_output import HyperoptOutput from freqtrade.optimize.hyperopt_tools import ( HyperoptStateContainer, HyperoptTools, @@ -47,6 +48,7 @@ from freqtrade.optimize.hyperopt_tools import ( ) from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver +from freqtrade.util import CustomProgress # Suppress scikit-learn FutureWarnings from skopt @@ -86,6 +88,8 @@ class Hyperopt: self.max_open_trades_space: List[Dimension] = [] self.dimensions: List[Dimension] = [] + self._hyper_out: HyperoptOutput = HyperoptOutput() + self.config = config self.min_date: datetime self.max_date: datetime @@ -260,7 +264,7 @@ class Hyperopt: result["max_open_trades"] = {"max_open_trades": strategy.max_open_trades} return result - def print_results(self, results) -> None: + def print_results(self, results: Dict[str, Any]) -> None: """ Log results if it is better than any previous evaluation TODO: this should be moved to HyperoptTools too @@ -268,17 +272,12 @@ class Hyperopt: is_best = results["is_best"] if self.print_all or is_best: - print( - HyperoptTools.get_result_table( - self.config, - results, - self.total_epochs, - self.print_all, - self.print_colorized, - self.hyperopt_table_header, - ) + self._hyper_out.add_data( + self.config, + [results], + self.total_epochs, + self.print_all, ) - self.hyperopt_table_header = 2 def init_spaces(self): """ @@ -626,16 +625,16 @@ class Hyperopt: self.opt = self.get_optimizer(self.dimensions, config_jobs) - if self.print_colorized: - colorama_init(autoreset=True) - try: with Parallel(n_jobs=config_jobs) as parallel: jobs = parallel._effective_n_jobs() logger.info(f"Effective number of parallel workers used: {jobs}") + console = Console( + color_system="auto" if self.print_colorized else None, + ) # Define progressbar - with Progress( + with CustomProgress( TextColumn("[progress.description]{task.description}"), BarColumn(bar_width=None), MofNCompleteColumn(), @@ -645,6 +644,8 @@ class Hyperopt: "•", TimeRemainingColumn(), expand=True, + console=console, + cust_objs=[Align.center(self._hyper_out.table)], ) as pbar: task = pbar.add_task("Epochs", total=self.total_epochs) diff --git a/freqtrade/optimize/hyperopt_output.py b/freqtrade/optimize/hyperopt_output.py new file mode 100644 index 000000000..72e049745 --- /dev/null +++ b/freqtrade/optimize/hyperopt_output.py @@ -0,0 +1,123 @@ +import sys +from typing import List, Optional, Union + +from rich.console import Console +from rich.table import Table +from rich.text import Text + +from freqtrade.constants import Config +from freqtrade.optimize.optimize_reports import generate_wins_draws_losses +from freqtrade.util import fmt_coin + + +class HyperoptOutput: + def __init__(self): + self.table = Table( + title="Hyperopt results", + ) + # Headers + self.table.add_column("Best", justify="left") + self.table.add_column("Epoch", justify="right") + self.table.add_column("Trades", justify="right") + self.table.add_column("Win Draw Loss Win%", justify="right") + self.table.add_column("Avg profit", justify="right") + self.table.add_column("Profit", justify="right") + self.table.add_column("Avg duration", justify="right") + self.table.add_column("Objective", justify="right") + self.table.add_column("Max Drawdown (Acct)", justify="right") + + def _add_row(self, data: List[Union[str, Text]]): + """Add single row""" + row_to_add: List[Union[str, Text]] = [r if isinstance(r, Text) else str(r) for r in data] + + self.table.add_row(*row_to_add) + + def _add_rows(self, data: List[List[Union[str, Text]]]): + """add multiple rows""" + for row in data: + self._add_row(row) + + def print(self, console: Optional[Console] = None, *, print_colorized=True): + if not console: + console = Console( + color_system="auto" if print_colorized else None, + width=200 if "pytest" in sys.modules else None, + ) + + console.print(self.table) + + def add_data( + self, + config: Config, + results: list, + total_epochs: int, + highlight_best: bool, + ) -> None: + """Format one or multiple rows and add them""" + stake_currency = config["stake_currency"] + + for r in results: + self.table.add_row( + *[ + # "Best": + ( + ("*" if r["is_initial_point"] or r["is_random"] else "") + + (" Best" if r["is_best"] else "") + ).lstrip(), + # "Epoch": + f"{r['current_epoch']}/{total_epochs}", + # "Trades": + str(r["results_metrics"]["total_trades"]), + # "Win Draw Loss Win%": + generate_wins_draws_losses( + r["results_metrics"]["wins"], + r["results_metrics"]["draws"], + r["results_metrics"]["losses"], + ), + # "Avg profit": + f"{r['results_metrics']['profit_mean']:.2%}" + if r["results_metrics"]["profit_mean"] is not None + else "--", + # "Profit": + Text( + "{} {}".format( + fmt_coin( + r["results_metrics"]["profit_total_abs"], + stake_currency, + keep_trailing_zeros=True, + ), + f"({r['results_metrics']['profit_total']:,.2%})".rjust(10, " "), + ) + if r["results_metrics"].get("profit_total_abs", 0) != 0.0 + else "--", + style=( + "green" + if r["results_metrics"].get("profit_total_abs", 0) > 0 + else "red" + ) + if not r["is_best"] + else "", + ), + # "Avg duration": + str(r["results_metrics"]["holding_avg"]), + # "Objective": + f"{r['loss']:,.5f}" if r["loss"] != 100000 else "N/A", + # "Max Drawdown (Acct)": + "{} {}".format( + fmt_coin( + r["results_metrics"]["max_drawdown_abs"], + stake_currency, + keep_trailing_zeros=True, + ), + (f"({r['results_metrics']['max_drawdown_account']:,.2%})").rjust(10, " "), + ) + if r["results_metrics"]["max_drawdown_account"] != 0.0 + else "--", + ], + style=" ".join( + [ + "bold gold1" if r["is_best"] and highlight_best else "", + "italic " if r["is_initial_point"] else "", + ] + ), + ) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 50c55c43d..975338cd5 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -5,10 +5,7 @@ from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Tuple import numpy as np -import pandas as pd import rapidjson -import tabulate -from colorama import Fore, Style from pandas import isna, json_normalize from freqtrade.constants import FTHYPT_FILEVERSION, Config @@ -16,8 +13,6 @@ from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, round_dict, safe_value_fallback2 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs -from freqtrade.optimize.optimize_reports import generate_wins_draws_losses -from freqtrade.util import fmt_coin logger = logging.getLogger(__name__) @@ -357,175 +352,6 @@ class HyperoptTools: + f"Objective: {results['loss']:.5f}" ) - @staticmethod - def prepare_trials_columns(trials: pd.DataFrame) -> pd.DataFrame: - trials["Best"] = "" - - if "results_metrics.winsdrawslosses" not in trials.columns: - # Ensure compatibility with older versions of hyperopt results - trials["results_metrics.winsdrawslosses"] = "N/A" - - has_account_drawdown = "results_metrics.max_drawdown_account" in trials.columns - if not has_account_drawdown: - # Ensure compatibility with older versions of hyperopt results - trials["results_metrics.max_drawdown_account"] = None - if "is_random" not in trials.columns: - trials["is_random"] = False - - # New mode, using backtest result for metrics - trials["results_metrics.winsdrawslosses"] = trials.apply( - lambda x: generate_wins_draws_losses( - x["results_metrics.wins"], x["results_metrics.draws"], x["results_metrics.losses"] - ), - axis=1, - ) - - trials = trials[ - [ - "Best", - "current_epoch", - "results_metrics.total_trades", - "results_metrics.winsdrawslosses", - "results_metrics.profit_mean", - "results_metrics.profit_total_abs", - "results_metrics.profit_total", - "results_metrics.holding_avg", - "results_metrics.max_drawdown_account", - "results_metrics.max_drawdown_abs", - "loss", - "is_initial_point", - "is_random", - "is_best", - ] - ] - - trials.columns = [ - "Best", - "Epoch", - "Trades", - " Win Draw Loss Win%", - "Avg profit", - "Total profit", - "Profit", - "Avg duration", - "max_drawdown_account", - "max_drawdown_abs", - "Objective", - "is_initial_point", - "is_random", - "is_best", - ] - - return trials - - @staticmethod - def get_result_table( - config: Config, - results: list, - total_epochs: int, - highlight_best: bool, - print_colorized: bool, - remove_header: int, - ) -> str: - """ - Log result table - """ - if not results: - return "" - - tabulate.PRESERVE_WHITESPACE = True - trials = json_normalize(results, max_level=1) - - trials = HyperoptTools.prepare_trials_columns(trials) - - trials["is_profit"] = False - trials.loc[trials["is_initial_point"] | trials["is_random"], "Best"] = "* " - trials.loc[trials["is_best"], "Best"] = "Best" - trials.loc[ - (trials["is_initial_point"] | trials["is_random"]) & trials["is_best"], "Best" - ] = "* Best" - trials.loc[trials["Total profit"] > 0, "is_profit"] = True - trials["Trades"] = trials["Trades"].astype(str) - # perc_multi = 1 if legacy_mode else 100 - trials["Epoch"] = trials["Epoch"].apply( - lambda x: "{}/{}".format(str(x).rjust(len(str(total_epochs)), " "), total_epochs) - ) - trials["Avg profit"] = trials["Avg profit"].apply( - lambda x: f"{x:,.2%}".rjust(7, " ") if not isna(x) else "--".rjust(7, " ") - ) - trials["Avg duration"] = trials["Avg duration"].apply( - lambda x: ( - f"{x:,.1f} m".rjust(7, " ") - if isinstance(x, float) - else f"{x}" - if not isna(x) - else "--".rjust(7, " ") - ) - ) - trials["Objective"] = trials["Objective"].apply( - lambda x: f"{x:,.5f}".rjust(8, " ") if x != 100000 else "N/A".rjust(8, " ") - ) - - stake_currency = config["stake_currency"] - - trials["Max Drawdown (Acct)"] = trials.apply( - lambda x: ( - "{} {}".format( - fmt_coin(x["max_drawdown_abs"], stake_currency, keep_trailing_zeros=True), - (f"({x['max_drawdown_account']:,.2%})").rjust(10, " "), - ).rjust(25 + len(stake_currency)) - if x["max_drawdown_account"] != 0.0 - else "--".rjust(25 + len(stake_currency)) - ), - axis=1, - ) - - trials = trials.drop(columns=["max_drawdown_abs", "max_drawdown_account"]) - - trials["Profit"] = trials.apply( - lambda x: ( - "{} {}".format( - fmt_coin(x["Total profit"], stake_currency, keep_trailing_zeros=True), - f"({x['Profit']:,.2%})".rjust(10, " "), - ).rjust(25 + len(stake_currency)) - if x["Total profit"] != 0.0 - else "--".rjust(25 + len(stake_currency)) - ), - axis=1, - ) - trials = trials.drop(columns=["Total profit"]) - - if print_colorized: - trials2 = trials.astype(str) - for i in range(len(trials)): - if trials.loc[i]["is_profit"]: - for j in range(len(trials.loc[i]) - 3): - trials2.iat[i, j] = f"{Fore.GREEN}{str(trials.iloc[i, j])}{Fore.RESET}" - if trials.loc[i]["is_best"] and highlight_best: - for j in range(len(trials.loc[i]) - 3): - trials2.iat[i, j] = ( - f"{Style.BRIGHT}{str(trials.iloc[i, j])}{Style.RESET_ALL}" - ) - trials = trials2 - del trials2 - trials = trials.drop(columns=["is_initial_point", "is_best", "is_profit", "is_random"]) - if remove_header > 0: - table = tabulate.tabulate( - trials.to_dict(orient="list"), tablefmt="orgtbl", headers="keys", stralign="right" - ) - - table = table.split("\n", remove_header)[remove_header] - elif remove_header < 0: - table = tabulate.tabulate( - trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right" - ) - table = "\n".join(table.split("\n")[0:remove_header]) - else: - table = tabulate.tabulate( - trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right" - ) - return table - @staticmethod def export_csv_file(config: Config, results: list, csv_file: str) -> None: """ diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 503f5861a..76902b176 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -15,6 +15,8 @@ from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.measure_time import MeasureTime from freqtrade.util.periodic_cache import PeriodicCache +from freqtrade.util.rich_progress import CustomProgress +from freqtrade.util.rich_tables import print_df_rich_table, print_rich_table from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa @@ -36,4 +38,7 @@ __all__ = [ "round_value", "fmt_coin", "MeasureTime", + "print_rich_table", + "print_df_rich_table", + "CustomProgress", ] diff --git a/freqtrade/util/rich_progress.py b/freqtrade/util/rich_progress.py new file mode 100644 index 000000000..d295dafd5 --- /dev/null +++ b/freqtrade/util/rich_progress.py @@ -0,0 +1,14 @@ +from typing import Union + +from rich.console import ConsoleRenderable, Group, RichCast +from rich.progress import Progress + + +class CustomProgress(Progress): + def __init__(self, *args, cust_objs, **kwargs) -> None: + self._cust_objs = cust_objs + super().__init__(*args, **kwargs) + + def get_renderable(self) -> Union[ConsoleRenderable, RichCast, str]: + renderable = Group(*self._cust_objs, *self.get_renderables()) + return renderable diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py new file mode 100644 index 000000000..d9762c9fb --- /dev/null +++ b/freqtrade/util/rich_tables.py @@ -0,0 +1,76 @@ +import sys +from typing import Any, Dict, List, Optional, Sequence, Union + +from pandas import DataFrame +from rich.console import Console +from rich.table import Column, Table +from rich.text import Text + + +TextOrString = Union[str, Text] + + +def print_rich_table( + tabular_data: Sequence[Union[Dict[str, Any], Sequence[TextOrString]]], + headers: Sequence[str], + summary: Optional[str] = None, + *, + table_kwargs: Optional[Dict[str, Any]] = None, +) -> None: + table = Table( + *[c if isinstance(c, Column) else Column(c, justify="right") for c in headers], + title=summary, + **(table_kwargs or {}), + ) + + for row in tabular_data: + if isinstance(row, dict): + table.add_row( + *[ + row[header] if isinstance(row[header], Text) else str(row[header]) + for header in headers + ] + ) + + else: + row_to_add: List[Union[str, Text]] = [r if isinstance(r, Text) else str(r) for r in row] + table.add_row(*row_to_add) + + console = Console( + width=200 if "pytest" in sys.modules else None, + ) + console.print(table) + + +def _format_value(value: Any, *, floatfmt: str) -> str: + if isinstance(value, float): + return f"{value:{floatfmt}}" + return str(value) + + +def print_df_rich_table( + tabular_data: DataFrame, + headers: Sequence[str], + summary: Optional[str] = None, + *, + show_index=False, + index_name: Optional[str] = None, + table_kwargs: Optional[Dict[str, Any]] = None, +) -> None: + table = Table(title=summary, **(table_kwargs or {})) + + if show_index: + index_name = str(index_name) if index_name else tabular_data.index.name + table.add_column(index_name) + + for header in headers: + table.add_column(header, justify="right") + + for value_list in tabular_data.itertuples(index=show_index): + row = [_format_value(x, floatfmt=".3f") for x in value_list] + table.add_row(*row) + + console = Console( + width=200 if "pytest" in sys.modules else None, + ) + console.print(table) diff --git a/requirements.txt b/requirements.txt index 994dc4d32..782cbcc2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,8 +45,6 @@ pyjwt==2.8.0 aiofiles==24.1.0 psutil==6.0.0 -# Support for colorized terminal output -colorama==0.4.6 # Building config files interactively questionary==2.0.1 prompt-toolkit==3.0.36 diff --git a/setup.py b/setup.py index 82e529767..6963862e0 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,6 @@ setup( "py_find_1st", "python-rapidjson", "orjson", - "colorama", "jinja2", "questionary", "prompt-toolkit", diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 02b234b6c..687bff69f 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -116,7 +116,7 @@ def test_list_exchanges(capsys): start_list_exchanges(get_args(args)) captured = capsys.readouterr() - assert re.match(r"Exchanges available for Freqtrade.*", captured.out) + assert re.search(r".*Exchanges available for Freqtrade.*", captured.out) assert re.search(r".*binance.*", captured.out) assert re.search(r".*bybit.*", captured.out) @@ -139,7 +139,7 @@ def test_list_exchanges(capsys): start_list_exchanges(get_args(args)) captured = capsys.readouterr() - assert re.match(r"All exchanges supported by the ccxt library.*", captured.out) + assert re.search(r"All exchanges supported by the ccxt library.*", captured.out) assert re.search(r".*binance.*", captured.out) assert re.search(r".*bingx.*", captured.out) assert re.search(r".*bitmex.*", captured.out) @@ -293,7 +293,7 @@ def test_list_markets(mocker, markets_static, capsys): pargs["config"] = None start_list_markets(pargs, False) captured = capsys.readouterr() - assert re.match("\nExchange Binance has 12 active markets:\n", captured.out) + assert re.search(r".*Exchange Binance has 12 active markets.*", captured.out) patch_exchange(mocker, api_mock=api_mock, exchange="binance", mock_markets=markets_static) # Test with --all: all markets @@ -491,7 +491,7 @@ def test_list_markets(mocker, markets_static, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert "Exchange Binance has 12 active markets:\n" in captured.out + assert "Exchange Binance has 12 active markets" in captured.out # Test tabular output, no markets found args = [ @@ -1633,8 +1633,8 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() assert "Found 16 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe | Type |\n" in captured.out - assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m | spot |\n" in captured.out + assert re.search(r".*Pair.*Timeframe.*Type.*\n", captured.out) + assert re.search(r"\n.* UNITTEST/BTC .* 1m, 5m, 8m, 30m .* spot |\n", captured.out) args = [ "list-data", @@ -1650,9 +1650,9 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() assert "Found 2 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe | Type |\n" in captured.out + assert re.search(r".*Pair.*Timeframe.*Type.*\n", captured.out) assert "UNITTEST/BTC" not in captured.out - assert "\n| XRP/ETH | 1m, 5m | spot |\n" in captured.out + assert re.search(r"\n.* XRP/ETH .* 1m, 5m .* spot |\n", captured.out) args = [ "list-data", @@ -1667,9 +1667,9 @@ def test_start_list_data(testdatadir, capsys): captured = capsys.readouterr() assert "Found 6 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe | Type |\n" in captured.out - assert "\n| XRP/USDT:USDT | 5m, 1h | futures |\n" in captured.out - assert "\n| XRP/USDT:USDT | 1h, 8h | mark |\n" in captured.out + assert re.search(r".*Pair.*Timeframe.*Type.*\n", captured.out) + assert re.search(r"\n.* XRP/USDT:USDT .* 5m, 1h .* futures |\n", captured.out) + assert re.search(r"\n.* XRP/USDT:USDT .* 1h, 8h .* mark |\n", captured.out) args = [ "list-data", @@ -1684,15 +1684,12 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() assert "Found 2 pair / timeframe combinations." in captured.out - assert ( - "\n| Pair | Timeframe | Type " - "| From | To | Candles |\n" - ) in captured.out + assert re.search(r".*Pair.*Timeframe.*Type.*From .* To .* Candles .*\n", captured.out) assert "UNITTEST/BTC" not in captured.out - assert ( - "\n| XRP/ETH | 1m | spot | " - "2019-10-11 00:00:00 | 2019-10-13 11:19:00 | 2469 |\n" - ) in captured.out + assert re.search( + r"\n.* XRP/USDT .* 1m .* spot .* 2019-10-11 00:00:00 .* 2019-10-13 11:19:00 .* 2469 |\n", + captured.out, + ) @pytest.mark.usefixtures("init_persistence") diff --git a/tests/conftest_hyperopt.py b/tests/conftest_hyperopt.py index af4039a3c..315b138cf 100644 --- a/tests/conftest_hyperopt.py +++ b/tests/conftest_hyperopt.py @@ -324,7 +324,8 @@ def hyperopt_test_result(): "profit_mean": None, "profit_median": None, "profit_total": 0, - "profit": 0.0, + "max_drawdown_account": 0.0, + "max_drawdown_abs": 0.0, "holding_avg": timedelta(), }, # noqa: E501 "results_explanation": " 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.", # noqa: E501 diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 1a5309190..e7909c339 100644 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -154,10 +154,10 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, use assert "-3.5" in captured.out assert "50" in captured.out assert "0" in captured.out - assert "0.01616" in captured.out + assert "0.016" in captured.out assert "34.049" in captured.out - assert "0.104411" in captured.out - assert "52.8292" in captured.out + assert "0.104" in captured.out + assert "52.829" in captured.out # test group 1 args = get_args(base_args + ["--analysis-groups", "1"]) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 2e9ae8d35..a9f697629 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -291,9 +291,10 @@ def test_log_results_if_loss_improves(hyperopt, capsys) -> None: "is_best": True, } ) + hyperopt._hyper_out.print() out, _err = capsys.readouterr() assert all( - x in out for x in ["Best", "2/2", " 1", "0.10%", "0.00100000 BTC (1.00%)", "00:20:00"] + x in out for x in ["Best", "2/2", "1", "0.10%", "0.00100000 BTC (1.00%)", "0:20:00"] ) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 88e3ad877..f7d38b24b 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -147,7 +147,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.current_analysis = analysis - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance] ) @@ -163,14 +163,14 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf analysis.false_exit_signals = 10 instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.current_analysis = analysis - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance] ) assert data[0][2].__contains__("error") # edit it into not showing an error instance.failed_bias_check = False - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance] ) assert data[0][0] == "strategy_test_v3_with_lookahead_bias.py" @@ -183,7 +183,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf analysis.false_indicators.append("falseIndicator1") analysis.false_indicators.append("falseIndicator2") - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance] ) @@ -193,7 +193,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf assert len(data) == 1 # check amount of multiple rows - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance, instance, instance] ) assert len(data) == 3 diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 2969b4153..e16c82d24 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -105,9 +105,7 @@ def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf instance = RecursiveAnalysis(recursive_conf, strategy_obj) instance.dict_recursive = dict_diff - _table, _headers, data = RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances( - [instance] - ) + data = RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances([instance]) # check row contents for a try that has too few signals assert data[0][0] == "rsi" @@ -118,9 +116,7 @@ def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf dict_diff = dict() instance = RecursiveAnalysis(recursive_conf, strategy_obj) instance.dict_recursive = dict_diff - _table, _headers, data = RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances( - [instance] - ) + data = RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances([instance]) assert len(data) == 0