From 6a1655c047b88bba462677566f6943819ffc83a7 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 15 Nov 2022 22:26:54 -0700 Subject: [PATCH 1/9] support ssl connections in emc --- freqtrade/constants.py | 1 + freqtrade/rpc/external_message_consumer.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 534d06fd4..cfac98ebd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -512,6 +512,7 @@ CONF_SCHEMA = { 'minimum': 0, 'maximum': 65535 }, + 'secure': {'type': 'boolean', 'default': False}, 'ws_token': {'type': 'string'}, }, 'required': ['name', 'host', 'ws_token'] diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index b978407e4..d9aed7d52 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -31,6 +31,7 @@ class Producer(TypedDict): name: str host: str port: int + secure: bool ws_token: str @@ -180,7 +181,8 @@ class ExternalMessageConsumer: host, port = producer['host'], producer['port'] token = producer['ws_token'] name = producer['name'] - ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + scheme = 'wss' if producer['secure'] else 'ws' + ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" # This will raise InvalidURI if the url is bad async with websockets.connect( From 86e094e39b77e9c6ea8b8e88fa7a84339be151e0 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 15 Nov 2022 22:33:42 -0700 Subject: [PATCH 2/9] update docs --- docs/producer-consumer.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/producer-consumer.md b/docs/producer-consumer.md index b69406edf..88e34d0d6 100644 --- a/docs/producer-consumer.md +++ b/docs/producer-consumer.md @@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect "name": "default", // This can be any name you'd like, default is "default" "host": "127.0.0.1", // The host from your producer's api_server config "port": 8080, // The port from your producer's api_server config + "secure": false, // Use a secure websockets connection, default false "ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config } ], @@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect | `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.
**Datatype:** string | `producers.host` | **Required.** The hostname or IP address from your producer.
**Datatype:** string | `producers.port` | **Required.** The port matching the above host.
**Datatype:** string +| `producers.secure` | **Optional.** Use ssl in websockets connection. Default False.
**Datatype:** string | `producers.ws_token` | **Required.** `ws_token` as configured on the producer.
**Datatype:** string | | **Optional settings** | `wait_timeout` | Timeout until we ping again if no message is received.
*Defaults to `300`.*
**Datatype:** Integer - in seconds. From 1380ddd066c46784ea0ffd518f919ef8da7972f5 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 15 Nov 2022 22:44:02 -0700 Subject: [PATCH 3/9] update ws client --- scripts/ws_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 090039cde..70dbead14 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -199,6 +199,7 @@ async def create_client( host, port, token, + scheme='ws', name='default', protocol=ClientProtocol(), sleep_time=10, @@ -211,13 +212,14 @@ async def create_client( :param host: The host :param port: The port :param token: The websocket auth token + :param scheme: `ws` for most connections, `wss` for ssl :param name: The name of the producer :param **kwargs: Any extra kwargs passed to websockets.connect """ while 1: try: - websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" logger.info(f"Attempting to connect to {name} @ {host}:{port}") async with websockets.connect(websocket_url, **kwargs) as ws: @@ -304,6 +306,7 @@ async def _main(args): producer['host'], producer['port'], producer['ws_token'], + 'wss' if producer['secure'] else 'ws', producer['name'], sleep_time=sleep_time, ping_timeout=ping_timeout, From a993cb512de6422d0c186e27c214502a5356c5eb Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Thu, 17 Nov 2022 10:22:55 -0700 Subject: [PATCH 4/9] change to get call in ws_client --- scripts/ws_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 70dbead14..5d27f512e 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -306,7 +306,7 @@ async def _main(args): producer['host'], producer['port'], producer['ws_token'], - 'wss' if producer['secure'] else 'ws', + 'wss' if producer.get('secure', False) else 'ws', producer['name'], sleep_time=sleep_time, ping_timeout=ping_timeout, From 106ac2ab4d76ab32ac25e8b77e429d7b22779bbc Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sun, 20 Nov 2022 16:36:22 -0700 Subject: [PATCH 5/9] fix tests, change to get call --- freqtrade/rpc/external_message_consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index d9aed7d52..6078efd07 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -181,7 +181,7 @@ class ExternalMessageConsumer: host, port = producer['host'], producer['port'] token = producer['ws_token'] name = producer['name'] - scheme = 'wss' if producer['secure'] else 'ws' + scheme = 'wss' if producer.get('secure', False) else 'ws' ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" # This will raise InvalidURI if the url is bad From edb817e2e6254b9156b00e89e8144ba54f1c9644 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sun, 20 Nov 2022 19:19:28 -0700 Subject: [PATCH 6/9] add tutorial for ssl in docs --- docs/rest-api.md | 169 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/docs/rest-api.md b/docs/rest-api.md index c7d762648..2087d9a94 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -458,3 +458,172 @@ The correct configuration for this case is `http://localhost:8080` - the main pa !!! Note We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. + + +### Using SSL/TLS + +SSL/TLS is used to provide security and encrypt network traffic. Freqtrade does not directly support SSL, but you can easily accomplish this with a reverse proxy such as Nginx or Traefik. Below are some steps to help you get started on setting one up for your bot. For the sake of simplicity, we will use a native installation of Nginx and certbot. + + +**Prerequisites** + +Before starting this tutorial, you will need a few things. + +- A Freqtrade bot set up and running +- A registered domain name, e.g. myftbot.com +- A DNS A record for the top level domain pointing to your server's public IP + +**Step 1: Installing Nginx and Certbot** + +Once you have all of the prerequisites, the first step is to get Nginx installed on your system. This tutorial assumes the use of Ubuntu 20.04, though you can find your linux distro's package commands via a search engine. First, update your local package index so that you have access to the most recent package listings then install Nginx: + +``` bash +> sudo apt update +> sudo apt install nginx +``` + +After accepting the installation, Nginx and any dependencies will be installed to your system and automatically started. You can check it is running with `systemd`: + +``` bash +> sudo systemctl status nginx + +● nginx.service - A high performance web server and a reverse proxy server + Loaded: loaded (/lib/systemd/system/nginx.service; enabled; vendor preset: enabled) + Active: active (running) since Wed 2022-11-16 12:09:27 MST; 4 days ago + Docs: man:nginx(8) + Process: 1026 ExecStartPre=/usr/sbin/nginx -t -q -g daemon on; master_process on; (code=exited, status=0/SUCCESS) + Process: 1106 ExecStart=/usr/sbin/nginx -g daemon on; master_process on; (code=exited, status=0/SUCCESS) + Main PID: 1107 (nginx) + Tasks: 5 (limit: 6929) + Memory: 5.7M + CGroup: /system.slice/nginx.service + ├─1107 nginx: master process /usr/sbin/nginx -g daemon on; master_process on; + ├─1108 nginx: worker process + ├─1109 nginx: worker process + ├─1110 nginx: worker process + └─1111 nginx: worker process +``` + +Next you need to install certbot which will handle all of the certificate automation for your web server and domain. To install certbot it is required to have `snapd` on Ubuntu. If you haven't installed it yet, please review the instructions on [snapcraft's site for installation](https://snapcraft.io/docs/installing-snapd/). + +Once you are good to go, ensure your snapd version is up to date: + +``` bash +> sudo snap install core; sudo snap refresh core +``` + +If you have any Certbot packages installed via your package manager, you should remove them before installing Certbot: + +``` bash +> sudo apt remove certbot +``` + +Finally, install Certbot and prepare the Certbot command. + +``` bash +> sudo snap install --classic certbot +> sudo ln -s /snap/bin/certbot /usr/bin/certbot +``` + +**Step 2: Adjust the firewall** + +The next step is to allow HTTP and HTTPS traffic through your firewall. This is different for each depending on which firewall you use, and how you have it configured. In this example, we are using `ufw`. + +We'll start by enabling `ufw` if it isn't already: + +``` bash +> sudo ufw enable +``` + +You can list the application configurations that ufw knows how to work with + +``` bash +> sudo ufw app list + +Available applications: + CUPS + Nginx Full + Nginx HTTP + Nginx HTTPS +``` + +As you can see in the output, there are 3 profiles available for Nginx: + +- **Nginx Full**: This profile opens both port 80 (normal web traffic) and port 443 (SSL/TLS traffic) +- **Nginx HTTP**: This profile only opens port 80 (normal web traffic) +- **Nginx HTTPS**: This profile only opens port 443 (SSL/TLS traffic) + +We will configure the firewall to allow both port 80 and 443: + +``` bash +> sudo ufw allow 'Nginx Full' +``` + +You can verify the change by typing: + +``` bash +> sudo ufw status + +Status: active + +To Action From +-- ------ ---- +Nginx HTTPS ALLOW Anywhere +Nginx Full ALLOW Anywhere +Nginx HTTPS (v6) ALLOW Anywhere (v6) +Nginx Full (v6) ALLOW Anywhere (v6) +``` + +**Step 3: Configuring Nginx** + +Using your favorite editor, edit the default nginx configuration. In our case, it'll be under `/etc/nginx/conf.d/default.conf`: +``` bash +> sudo vim /etc/nginx/conf.d/default.conf +``` + +Add a section to your configuration like this: + +``` +server { + server_name myftbot.com; + location / { + proxy_pass http://localhost:8080; + } +} +``` + +Make sure to change `localhost` and `8080` to what you have set in your `api_server` configuration for your bot. + +Verify your nginx config file syntax and make sure there are no errors: +``` bash +> sudo nginx -t +``` + +Finally you can reload nginx to get the new configuration changes: + +``` bash +> sudo systemctl reload nginx +``` + +!!! Note + The `reload` command forces Nginx to read the new configuration without interrupting any connections. The `restart` command restarts the whole nginx service. + +**Step 4: Getting the certificates** + +Certbot already comes with an easy way to setup Nginx with SSL/TLS th automatically changes your configuration file with the required fields: + +``` bash +> sudo certbot --nginx +``` + +You will be prompted for some information such as your email (To receive updates about your certificates), the domain you pointed to the server, and agree to the TOS and optional newsletter. You can also set to redirect HTTP traffic to HTTPS, removing HTTP access. + +You can now test your SSL setup by using curl to make a request to your bot's Rest API: + +``` bash +> curl https://myftbot.com/api/v1/ping + +{'status': 'pong'} +``` + +If you see a pong response then everything is working and you have successfully set up SSL/TLS termination for your bot. From 86ff711525ba13cd88673686f90eed994e18e8b9 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Mon, 21 Nov 2022 12:52:18 -0700 Subject: [PATCH 7/9] update docs on reverse proxy --- docs/rest-api.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index 2087d9a94..beb9ac194 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -389,6 +389,23 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th } ``` +#### Reverse Proxy and Websockets + +There are some quirks when using a reverse proxy with the message websocket endpoint. The message websocket endpoint keeps a long-running connection open between the Rest API and the client. It's built on top of HTTP and uses the HTTP Upgrade mechanism to change from HTTP to WebSockets during connection. There are some challenges that a reverse proxy faces when supporting WebSockets, such as WebSockets are a hop-by-hop protocol, so when a proxy intercepts an Upgrade request from the client it needs to send it's own Upgrade request to the server, including appropriate headers. Also, since these connections are long lived, the proxy needs to allow these connections to remain open. + +When using Nginx, the following configuration is required for WebSockets to work: +``` +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $connection_upgrade; +proxy_set_header Host $host; +``` + +To configure your reverse proxy, see it's documentation for proxying websockets. + +- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/) +- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy) + ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. @@ -459,7 +476,7 @@ The correct configuration for this case is `http://localhost:8080` - the main pa !!! Note We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. - + From fff745fd83e63c5816801a000ac01ec279b6038e Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 22 Nov 2022 07:17:57 -0700 Subject: [PATCH 8/9] add map to nginx config --- docs/rest-api.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index beb9ac194..c7c41d571 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -393,12 +393,27 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th There are some quirks when using a reverse proxy with the message websocket endpoint. The message websocket endpoint keeps a long-running connection open between the Rest API and the client. It's built on top of HTTP and uses the HTTP Upgrade mechanism to change from HTTP to WebSockets during connection. There are some challenges that a reverse proxy faces when supporting WebSockets, such as WebSockets are a hop-by-hop protocol, so when a proxy intercepts an Upgrade request from the client it needs to send it's own Upgrade request to the server, including appropriate headers. Also, since these connections are long lived, the proxy needs to allow these connections to remain open. -When using Nginx, the following configuration is required for WebSockets to work: +When using Nginx, the following configuration is required for WebSockets to work (Note this configuration isn't complete, it's missing some information and can not be used as is): ``` -proxy_http_version 1.1; -proxy_set_header Upgrade $http_upgrade; -proxy_set_header Connection $connection_upgrade; -proxy_set_header Host $host; +http { + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + ... + + server { + ... + + location / { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + } + } +} ``` To configure your reverse proxy, see it's documentation for proxying websockets. From bd05f85c26bc65228af6b3942bbd1b2f3955eb3e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Nov 2022 18:11:18 +0100 Subject: [PATCH 9/9] Simplify ssl documentation --- docs/rest-api.md | 187 +++-------------------------------------------- 1 file changed, 12 insertions(+), 175 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index c7c41d571..62ad586dd 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -389,11 +389,12 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th } ``` -#### Reverse Proxy and Websockets +#### Reverse Proxy setup -There are some quirks when using a reverse proxy with the message websocket endpoint. The message websocket endpoint keeps a long-running connection open between the Rest API and the client. It's built on top of HTTP and uses the HTTP Upgrade mechanism to change from HTTP to WebSockets during connection. There are some challenges that a reverse proxy faces when supporting WebSockets, such as WebSockets are a hop-by-hop protocol, so when a proxy intercepts an Upgrade request from the client it needs to send it's own Upgrade request to the server, including appropriate headers. Also, since these connections are long lived, the proxy needs to allow these connections to remain open. +When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is): + +Please make sure to replace `` (and the subsequent port) with the IP and Port matching your configuration/setup. -When using Nginx, the following configuration is required for WebSockets to work (Note this configuration isn't complete, it's missing some information and can not be used as is): ``` http { map $http_upgrade $connection_upgrade { @@ -401,13 +402,14 @@ http { '' close; } - ... + #... server { - ... + #... location / { proxy_http_version 1.1; + proxy_pass http://:8080; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header Host $host; @@ -416,11 +418,15 @@ http { } ``` -To configure your reverse proxy, see it's documentation for proxying websockets. +To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets. - **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/) - **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy) +!!! Tip "SSL certificates" + You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies. + While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel). + ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. @@ -490,172 +496,3 @@ The correct configuration for this case is `http://localhost:8080` - the main pa !!! Note We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. - -