Merge pull request #5083 from freqtrade/test_multiarch

Combine docker build scripts
This commit is contained in:
Matthias 2021-06-06 18:19:58 +01:00 committed by GitHub
commit be6d6b7d74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 51 additions and 86 deletions

View File

@ -75,7 +75,7 @@ jobs:
COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu
run: |
# Allow failure for coveralls
coveralls -v || true
coveralls || true
- name: Backtesting
run: |
@ -392,19 +392,12 @@ jobs:
- name: Available platforms
run: echo ${{ steps.buildx.outputs.platforms }}
- name: Build and test and push docker image
- name: Build and test and push docker images
env:
IMAGE_NAME: freqtradeorg/freqtrade
BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
run: |
build_helpers/publish_docker.sh
- name: Build Raspberry docker image
env:
IMAGE_NAME: freqtradeorg/freqtrade
BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
run: |
build_helpers/publish_docker_pi.sh
build_helpers/publish_docker_multi.sh
- name: Slack Notification

View File

@ -46,12 +46,6 @@ jobs:
- script: mypy freqtrade scripts
name: mypy
# - stage: docker
# if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron))
# script:
# - build_helpers/publish_docker.sh
# name: "Build and test and push docker image"
notifications:
slack:
secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q=

View File

@ -1,21 +1,48 @@
#!/bin/sh
# The below assumes a correctly setup docker buildx environment
# Replace / with _ to create a valid tag
TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
TAG_PLOT=${TAG}_plot
TAG_PI="${TAG}_pi"
PI_PLATFORM="linux/arm/v7"
echo "Running for ${TAG}"
CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache
# Add commit and commit_message to docker container
echo "${GITHUB_SHA}" > freqtrade_commit
if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache"
# Build regular image
docker build -t freqtrade:${TAG} .
# Build PI image
docker buildx build \
--cache-to=type=registry,ref=${CACHE_TAG} \
-f docker/Dockerfile.armhf \
--platform ${PI_PLATFORM} \
-t ${IMAGE_NAME}:${TAG_PI} --push .
else
echo "event ${GITHUB_EVENT_NAME}: building with cache"
# Pull last build to avoid rebuilding the whole image
# Build regular image
docker pull ${IMAGE_NAME}:${TAG}
docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} .
# Pull last build to avoid rebuilding the whole image
# docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG}
docker buildx build \
--cache-from=type=registry,ref=${CACHE_TAG} \
--cache-to=type=registry,ref=${CACHE_TAG} \
-f docker/Dockerfile.armhf \
--platform ${PI_PLATFORM} \
-t ${IMAGE_NAME}:${TAG_PI} --push .
fi
if [ $? -ne 0 ]; then
echo "failed building multiarch images"
return 1
fi
# Tag image for upload and next build step
docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG
@ -24,11 +51,6 @@ docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t fre
docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT
if [ $? -ne 0 ]; then
echo "failed building image"
return 1
fi
# Run backtest
docker run --rm -v $(pwd)/config_bittrex.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy
@ -37,24 +59,29 @@ if [ $? -ne 0 ]; then
return 1
fi
if [ $? -ne 0 ]; then
echo "failed tagging image"
return 1
fi
# Tag as latest for develop builds
if [ "${TAG}" = "develop" ]; then
docker tag freqtrade:$TAG ${IMAGE_NAME}:latest
fi
# Show all available images
docker images
docker push ${IMAGE_NAME}
docker push ${IMAGE_NAME}:$TAG_PLOT
docker push ${IMAGE_NAME}:$TAG
# Create multiarch image
# Make sure that all images contained here are pushed to github first.
# Otherwise installation might fail.
docker manifest create freqtradeorg/freqtrade:${TAG} ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI}
docker manifest push freqtradeorg/freqtrade:${TAG}
# Tag as latest for develop builds
if [ "${TAG}" = "develop" ]; then
docker manifest create freqtradeorg/freqtrade:latest ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI}
docker manifest push freqtradeorg/freqtrade:latest
fi
docker images
if [ $? -ne 0 ]; then
echo "failed pushing repo"
echo "failed building image"
return 1
fi

View File

@ -1,49 +0,0 @@
#!/bin/sh
# The below assumes a correctly setup docker buildx environment
# Replace / with _ to create a valid tag
TAG_ORIG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
TAG="${TAG_ORIG}_pi"
PI_PLATFORM="linux/arm/v7"
echo "Running for ${TAG}"
CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache
# Add commit and commit_message to docker container
echo "${GITHUB_SHA}" > freqtrade_commit
if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache"
docker buildx build \
--cache-to=type=registry,ref=${CACHE_TAG} \
-f docker/Dockerfile.armhf \
--platform ${PI_PLATFORM} \
-t ${IMAGE_NAME}:${TAG} --push .
else
echo "event ${GITHUB_EVENT_NAME}: building with cache"
# Pull last build to avoid rebuilding the whole image
# docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG}
docker buildx build \
--cache-from=type=registry,ref=${CACHE_TAG} \
--cache-to=type=registry,ref=${CACHE_TAG} \
-f docker/Dockerfile.armhf \
--platform ${PI_PLATFORM} \
-t ${IMAGE_NAME}:${TAG} --push .
fi
docker images
# Create multiarch image
# Make sure that all images contained here are pushed to github first.
# Otherwise installation might fail.
docker manifest create freqtradeorg/freqtrade:${TAG_ORIG} ${IMAGE_NAME}:${TAG_ORIG} ${IMAGE_NAME}:${TAG}
docker manifest push freqtradeorg/freqtrade:${TAG_ORIG}
docker images
if [ $? -ne 0 ]; then
echo "failed building image"
return 1
fi

View File

@ -1,4 +1,4 @@
FROM --platform=linux/arm/v7 python:3.7.10-slim-buster as base
FROM python:3.7.10-slim-buster as base
# Setup env
ENV LANG C.UTF-8
@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker"
# Prepare environment
RUN mkdir /freqtrade \
&& apt-get update \
&& apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \
&& apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev \
&& apt-get clean \
&& useradd -u 1000 -G sudo -U -m ftuser \
&& chown ftuser:ftuser /freqtrade \
@ -23,7 +23,7 @@ WORKDIR /freqtrade
# Install dependencies
FROM base as python-deps
RUN apt-get update \
&& apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \
&& apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 pkg-config cmake gcc \
&& apt-get clean \
&& pip install --upgrade pip \
&& echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf