lychiyu/freqtrade_origin
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2024-11-15 12:43:56 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity
a103da314c
freqtrade_origin/en/2023.1/freqai-developers/index.html

1394 lines
36 KiB
HTML
Raw Blame History

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="canonical" href="https://www.freqtrade.io/2023.1/freqai-developers/">
<link rel="prev" href="../freqai-reinforcement-learning/">
<link rel="next" href="../leverage/">
<link rel="icon" href="../images/logo.png">
<meta name="generator" content="mkdocs-1.4.2, mkdocs-material-9.0.5">
<title>Developer guide - Freqtrade</title>
<link rel="stylesheet" href="../assets/stylesheets/main.558e4712.min.css">
<link rel="stylesheet" href="../assets/stylesheets/palette.2505c338.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<link rel="stylesheet" href="../stylesheets/ft.extra.css">
<script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="blue-grey" data-md-color-accent="tear">
<script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#development" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<div data-md-component="outdated" hidden>
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href=".." title="Freqtrade" class="md-header__button md-logo" aria-label="Freqtrade" data-md-component="logo">
<img src="../images/logo.png" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
Freqtrade
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Developer guide
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="blue-grey" data-md-color-accent="tear" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 6H7c-3.31 0-6 2.69-6 6s2.69 6 6 6h10c3.31 0 6-2.69 6-6s-2.69-6-6-6zm0 10H7c-2.21 0-4-1.79-4-4s1.79-4 4-4h10c2.21 0 4 1.79 4 4s-1.79 4-4 4zM7 9c-1.66 0-3 1.34-3 3s1.34 3 3 3 3-1.34 3-3-1.34-3-3-3z"/></svg>
</label>
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="blue-grey" data-md-color-accent="tear" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_2">
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3Z"/></svg>
</label>
</form>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<a href="javascript:void(0)" class="md-search__icon md-icon" title="Share" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7 0-.24-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91 1.61 0 2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08Z"/></svg>
</a>
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
</button>
</nav>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://github.com/freqtrade/freqtrade" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.2.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<!-- Main navigation -->
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href=".." title="Freqtrade" class="md-nav__button md-logo" aria-label="Freqtrade" data-md-component="logo">
<img src="../images/logo.png" alt="logo">
</a>
Freqtrade
</label>
<div class="md-nav__source">
<a href="https://github.com/freqtrade/freqtrade" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.2.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
</div>
<div class="md-source__repository">
GitHub
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href=".." class="md-nav__link">
Home
</a>
</li>
<li class="md-nav__item">
<a href="../docker_quickstart/" class="md-nav__link">
Quickstart with Docker
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " data-md-toggle="__nav_3" type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" tabindex="0" aria-expanded="false">
Installation
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Installation" data-md-level="1">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Installation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../installation/" class="md-nav__link">
Linux/MacOS/Raspberry
</a>
</li>
<li class="md-nav__item">
<a href="../windows_installation/" class="md-nav__link">
Windows
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../bot-basics/" class="md-nav__link">
Freqtrade Basics
</a>
</li>
<li class="md-nav__item">
<a href="../configuration/" class="md-nav__link">
Configuration
</a>
</li>
<li class="md-nav__item">
<a href="../strategy-customization/" class="md-nav__link">
Strategy Customization
</a>
</li>
<li class="md-nav__item">
<a href="../strategy-callbacks/" class="md-nav__link">
Strategy Callbacks
</a>
</li>
<li class="md-nav__item">
<a href="../stoploss/" class="md-nav__link">
Stoploss
</a>
</li>
<li class="md-nav__item">
<a href="../plugins/" class="md-nav__link">
Plugins
</a>
</li>
<li class="md-nav__item">
<a href="../bot-usage/" class="md-nav__link">
Start the bot
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " data-md-toggle="__nav_11" type="checkbox" id="__nav_11" >
<label class="md-nav__link" for="__nav_11" tabindex="0" aria-expanded="false">
Control the bot
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Control the bot" data-md-level="1">
<label class="md-nav__title" for="__nav_11">
<span class="md-nav__icon md-icon"></span>
Control the bot
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../telegram-usage/" class="md-nav__link">
Telegram
</a>
</li>
<li class="md-nav__item">
<a href="../rest-api/" class="md-nav__link">
REST API & FreqUI
</a>
</li>
<li class="md-nav__item">
<a href="../webhook-config/" class="md-nav__link">
Web Hook
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../data-download/" class="md-nav__link">
Data Downloading
</a>
</li>
<li class="md-nav__item">
<a href="../backtesting/" class="md-nav__link">
Backtesting
</a>
</li>
<li class="md-nav__item">
<a href="../hyperopt/" class="md-nav__link">
Hyperopt
</a>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle " data-md-toggle="__nav_15" type="checkbox" id="__nav_15" checked>
<label class="md-nav__link" for="__nav_15" tabindex="0" aria-expanded="true">
FreqAI
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="FreqAI" data-md-level="1">
<label class="md-nav__title" for="__nav_15">
<span class="md-nav__icon md-icon"></span>
FreqAI
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../freqai/" class="md-nav__link">
Introduction
</a>
</li>
<li class="md-nav__item">
<a href="../freqai-configuration/" class="md-nav__link">
Configuration
</a>
</li>
<li class="md-nav__item">
<a href="../freqai-parameter-table/" class="md-nav__link">
Parameter table
</a>
</li>
<li class="md-nav__item">
<a href="../freqai-feature-engineering/" class="md-nav__link">
Feature engineering
</a>
</li>
<li class="md-nav__item">
<a href="../freqai-running/" class="md-nav__link">
Running FreqAI
</a>
</li>
<li class="md-nav__item">
<a href="../freqai-reinforcement-learning/" class="md-nav__link">
Reinforcement Learning
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
Developer guide
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
Developer guide
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#project-architecture" class="md-nav__link">
Project architecture
</a>
</li>
<li class="md-nav__item">
<a href="#data-handling" class="md-nav__link">
Data handling
</a>
<nav class="md-nav" aria-label="Data handling">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#file-structure" class="md-nav__link">
File structure
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../leverage/" class="md-nav__link">
Short / Leverage
</a>
</li>
<li class="md-nav__item">
<a href="../utils/" class="md-nav__link">
Utility Sub-commands
</a>
</li>
<li class="md-nav__item">
<a href="../plotting/" class="md-nav__link">
Plotting
</a>
</li>
<li class="md-nav__item">
<a href="../exchanges/" class="md-nav__link">
Exchange-specific Notes
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " data-md-toggle="__nav_20" type="checkbox" id="__nav_20" >
<label class="md-nav__link" for="__nav_20" tabindex="0" aria-expanded="false">
Data Analysis
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Data Analysis" data-md-level="1">
<label class="md-nav__title" for="__nav_20">
<span class="md-nav__icon md-icon"></span>
Data Analysis
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../data-analysis/" class="md-nav__link">
Jupyter Notebooks
</a>
</li>
<li class="md-nav__item">
<a href="../strategy_analysis_example/" class="md-nav__link">
Strategy analysis
</a>
</li>
<li class="md-nav__item">
<a href="../advanced-backtesting/" class="md-nav__link">
Backtest analysis
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " data-md-toggle="__nav_21" type="checkbox" id="__nav_21" >
<label class="md-nav__link" for="__nav_21" tabindex="0" aria-expanded="false">
Advanced Topics
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" aria-label="Advanced Topics" data-md-level="1">
<label class="md-nav__title" for="__nav_21">
<span class="md-nav__icon md-icon"></span>
Advanced Topics
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../advanced-setup/" class="md-nav__link">
Advanced Post-installation Tasks
</a>
</li>
<li class="md-nav__item">
<a href="../trade-object/" class="md-nav__link">
Trade Object
</a>
</li>
<li class="md-nav__item">
<a href="../strategy-advanced/" class="md-nav__link">
Advanced Strategy
</a>
</li>
<li class="md-nav__item">
<a href="../advanced-hyperopt/" class="md-nav__link">
Advanced Hyperopt
</a>
</li>
<li class="md-nav__item">
<a href="../producer-consumer/" class="md-nav__link">
Producer/Consumer mode
</a>
</li>
<li class="md-nav__item">
<a href="../edge/" class="md-nav__link">
Edge Positioning
</a>
</li>
<li class="md-nav__item">
<a href="../sandbox-testing/" class="md-nav__link">
Sandbox Testing
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../faq/" class="md-nav__link">
FAQ
</a>
</li>
<li class="md-nav__item">
<a href="../sql_cheatsheet/" class="md-nav__link">
SQL Cheat-sheet
</a>
</li>
<li class="md-nav__item">
<a href="../strategy_migration/" class="md-nav__link">
Strategy migration
</a>
</li>
<li class="md-nav__item">
<a href="../updating/" class="md-nav__link">
Updating Freqtrade
</a>
</li>
<li class="md-nav__item">
<a href="../deprecated/" class="md-nav__link">
Deprecated Features
</a>
</li>
<li class="md-nav__item">
<a href="../developer/" class="md-nav__link">
Contributors Guide
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<!-- Table of contents -->
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#project-architecture" class="md-nav__link">
Project architecture
</a>
</li>
<li class="md-nav__item">
<a href="#data-handling" class="md-nav__link">
Data handling
</a>
<nav class="md-nav" aria-label="Data handling">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#file-structure" class="md-nav__link">
File structure
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="development">Development<a class="headerlink" href="#development" title="Permanent link">&para;</a></h1>
<h2 id="project-architecture">Project architecture<a class="headerlink" href="#project-architecture" title="Permanent link">&para;</a></h2>
<p>The architecture and functions of FreqAI are generalized to encourages development of unique features, functions, models, etc.</p>
<p>The class structure and a detailed algorithmic overview is depicted in the following diagram:</p>
<p><img alt="image" src="../assets/freqai_algorithm-diagram.jpg" /></p>
<p>As shown, there are three distinct objects comprising FreqAI:</p>
<ul>
<li><strong>IFreqaiModel</strong> - A singular persistent object containing all the necessary logic to collect, store, and process data, engineer features, run training, and inference models.</li>
<li><strong>FreqaiDataKitchen</strong> - A non-persistent object which is created uniquely for each unique asset/model. Beyond metadata, it also contains a variety of data processing tools.</li>
<li><strong>FreqaiDataDrawer</strong> - A singular persistent object containing all the historical predictions, models, and save/load methods.</li>
</ul>
<p>There are a variety of built-in <a href="../freqai-configuration/#using-different-prediction-models">prediction models</a> which inherit directly from <code>IFreqaiModel</code>. Each of these models have full access to all methods in <code>IFreqaiModel</code> and can therefore override any of those functions at will. However, advanced users will likely stick to overriding <code>fit()</code>, <code>train()</code>, <code>predict()</code>, and <code>data_cleaning_train/predict()</code>.</p>
<h2 id="data-handling">Data handling<a class="headerlink" href="#data-handling" title="Permanent link">&para;</a></h2>
<p>FreqAI aims to organize model files, prediction data, and meta data in a way that simplifies post-processing and enhances crash resilience by automatic data reloading. The data is saved in a file structure,<code>user_data_dir/models/</code>, which contains all the data associated with the trainings and backtests. The <code>FreqaiDataKitchen()</code> relies heavily on the file structure for proper training and inferencing and should therefore not be manually modified.</p>
<h3 id="file-structure">File structure<a class="headerlink" href="#file-structure" title="Permanent link">&para;</a></h3>
<p>The file structure is automatically generated based on the model <code>identifier</code> set in the <a href="../freqai-configuration/#setting-up-the-configuration-file">config</a>. The following structure shows where the data is stored for post processing:</p>
<table>
<thead>
<tr>
<th>Structure</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>config_*.json</code></td>
<td>A copy of the model specific configuration file.</td>
</tr>
<tr>
<td><code>historic_predictions.pkl</code></td>
<td>A file containing all historic predictions generated during the lifetime of the <code>identifier</code> model during live deployment. <code>historic_predictions.pkl</code> is used to reload the model after a crash or a config change. A backup file is always held in case of corruption on the main file. FreqAI <strong>automatically</strong> detects corruption and replaces the corrupted file with the backup.</td>
</tr>
<tr>
<td><code>pair_dictionary.json</code></td>
<td>A file containing the training queue as well as the on disk location of the most recently trained model.</td>
</tr>
<tr>
<td><code>sub-train-*_TIMESTAMP</code></td>
<td>A folder containing all the files associated with a single model, such as: <br></td>
</tr>
<tr>
<td></td>
<td><code>*_metadata.json</code> - Metadata for the model, such as normalization max/min, expected training feature list, etc. <br></td>
</tr>
<tr>
<td></td>
<td><code>*_model.*</code> - The model file saved to disk for reloading from a crash. Can be <code>joblib</code> (typical boosting libs), <code>zip</code> (stable_baselines), <code>hd5</code> (keras type), etc. <br></td>
</tr>
<tr>
<td></td>
<td><code>*_pca_object.pkl</code> - The <a href="../freqai-feature-engineering/#data-dimensionality-reduction-with-principal-component-analysis">Principal component analysis (PCA)</a> transform (if <code>principal_component_analysis: True</code> is set in the config) which will be used to transform unseen prediction features. <br></td>
</tr>
<tr>
<td></td>
<td><code>*_svm_model.pkl</code> - The <a href="../freqai-feature-engineering/#identifying-outliers-using-a-support-vector-machine-svm">Support Vector Machine (SVM)</a> model (if <code>use_SVM_to_remove_outliers: True</code> is set in the config) which is used to detect outliers in unseen prediction features. <br></td>
</tr>
<tr>
<td></td>
<td><code>*_trained_df.pkl</code> - The dataframe containing all the training features used to train the <code>identifier</code> model. This is used for computing the <a href="../freqai-feature-engineering/#identifying-outliers-with-the-dissimilarity-index-di">Dissimilarity Index (DI)</a> and can also be used for post-processing. <br></td>
</tr>
<tr>
<td></td>
<td><code>*_trained_dates.df.pkl</code> - The dates associated with the <code>trained_df.pkl</code>, which is useful for post-processing.</td>
</tr>
</tbody>
</table>
<p>The example file structure would look like this:</p>
<div class="highlight"><pre><span></span><code>├── models
│   └── unique-id
│   ├── config_freqai.example.json
│   ├── historic_predictions.backup.pkl
│   ├── historic_predictions.pkl
│   ├── pair_dictionary.json
│   ├── sub-train-1INCH_1662821319
│   │   ├── cb_1inch_1662821319_metadata.json
│   │   ├── cb_1inch_1662821319_model.joblib
│   │   ├── cb_1inch_1662821319_pca_object.pkl
│   │   ├── cb_1inch_1662821319_svm_model.joblib
│   │   ├── cb_1inch_1662821319_trained_dates_df.pkl
│   │   └── cb_1inch_1662821319_trained_df.pkl
│   ├── sub-train-1INCH_1662821371
│   │   ├── cb_1inch_1662821371_metadata.json
│   │   ├── cb_1inch_1662821371_model.joblib
│   │   ├── cb_1inch_1662821371_pca_object.pkl
│   │   ├── cb_1inch_1662821371_svm_model.joblib
│   │   ├── cb_1inch_1662821371_trained_dates_df.pkl
│   │   └── cb_1inch_1662821371_trained_df.pkl
│   ├── sub-train-ADA_1662821344
│   │   ├── cb_ada_1662821344_metadata.json
│   │   ├── cb_ada_1662821344_model.joblib
│   │   ├── cb_ada_1662821344_pca_object.pkl
│   │   ├── cb_ada_1662821344_svm_model.joblib
│   │   ├── cb_ada_1662821344_trained_dates_df.pkl
│   │   └── cb_ada_1662821344_trained_df.pkl
│   └── sub-train-ADA_1662821399
│   ├── cb_ada_1662821399_metadata.json
│   ├── cb_ada_1662821399_model.joblib
│   ├── cb_ada_1662821399_pca_object.pkl
│   ├── cb_ada_1662821399_svm_model.joblib
│   ├── cb_ada_1662821399_trained_dates_df.pkl
│   └── cb_ada_1662821399_trained_df.pkl
</code></pre></div>
</article>
</div>
</div>
<a href="#" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12Z"/></svg>
Back to top
</a>
</main>
<footer class="md-footer">
<nav class="md-footer__inner md-grid" aria-label="Footer" >
<a href="../freqai-reinforcement-learning/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Reinforcement Learning" rel="prev">
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
</div>
<div class="md-footer__title">
<div class="md-ellipsis">
<span class="md-footer__direction">
Previous
</span>
Reinforcement Learning
</div>
</div>
</a>
<a href="../leverage/" class="md-footer__link md-footer__link--next" aria-label="Next: Short / Leverage" rel="next">
<div class="md-footer__title">
<div class="md-ellipsis">
<span class="md-footer__direction">
Next
</span>
Short / Leverage
</div>
</div>
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4Z"/></svg>
</div>
</a>
</nav>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
<!-- Place this tag in your head or just before your close body tag. -->
<script async defer src="https://buttons.github.io/buttons.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js"
integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"base": "..", "features": ["content.code.annotate", "search.share", "content.code.copy", "navigation.top", "navigation.footer"], "search": "../assets/javascripts/workers/search.e5c33ebb.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": {"provider": "mike"}}</script>
<script src="../assets/javascripts/bundle.51d95adb.min.js"></script>
<script src="../javascripts/config.js"></script>
<script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
<script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink