diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..04e52c0fd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,208 @@ +name: Freqtrade CI + +on: + push: + branches: + - master + - develop + - github_actions_tests + tags: + pull_request: + schedule: + - cron: '0 5 * * 4' + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-18.04, macos-latest ] + python-version: [3.7] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache_dependencies + uses: actions/cache@v1 + id: cache + with: + path: ~/dependencies/ + key: ${{ runner.os }}-dependencies + + - name: pip cache (linux) + uses: actions/cache@preview + if: startsWith(matrix.os, 'ubuntu') + with: + path: ~/.cache/pip + key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip + + - name: pip cache (macOS) + uses: actions/cache@preview + if: startsWith(matrix.os, 'macOS') + with: + path: ~/Library/Caches/pip + key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip + + - name: TA binary *nix + if: steps.cache.outputs.cache-hit != 'true' + run: | + cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. + + - name: Installation - *nix + run: | + python -m pip install --upgrade pip + export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH + export TA_LIBRARY_PATH=${HOME}/dependencies/lib + export TA_INCLUDE_PATH=${HOME}/dependencies/include + pip install -r requirements-dev.txt + pip install -e . + + - name: Tests + run: | + pytest --random-order --cov=freqtrade --cov-config=.coveragerc + # Allow failure for coveralls + coveralls || true + + - name: Backtesting + run: | + cp config.json.example config.json + freqtrade backtesting --datadir tests/testdata --strategy DefaultStrategy + + - name: Hyperopt + run: | + cp config.json.example config.json + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt + + - name: Flake8 + run: | + flake8 + + - name: Mypy + run: | + mypy freqtrade scripts + + - name: Documentation syntax + run: | + ./tests/test_docs.sh + + - name: Slack Notification + uses: homoluctus/slatify@v1.8.0 + if: always() + with: + type: ${{ job.status }} + job_name: '*Freqtrade CI ${{ matrix.os }}*' + mention: 'here' + mention_if: 'failure' + channel: '#notifications' + url: ${{ secrets.SLACK_WEBHOOK }} + + build_windows: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ windows-latest ] + python-version: [3.7] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Pip cache (Windows) + uses: actions/cache@preview + if: startsWith(runner.os, 'Windows') + with: + path: ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-pip + restore-keys: ${{ runner.os }}-pip + + - name: Installation + run: | + ./build_helpers/install_windows.ps1 + + - name: Tests + run: | + pytest --random-order --cov=freqtrade --cov-config=.coveragerc + + - name: Backtesting + run: | + cp config.json.example config.json + freqtrade backtesting --datadir tests/testdata --strategy DefaultStrategy + + - name: Hyperopt + run: | + cp config.json.example config.json + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt + + - name: Flake8 + run: | + flake8 + + - name: Mypy + run: | + mypy freqtrade scripts + + - name: Slack Notification + uses: homoluctus/slatify@v1.8.0 + if: always() + with: + type: ${{ job.status }} + job_name: '*Freqtrade CI windows*' + mention: 'here' + mention_if: 'failure' + channel: '#notifications' + url: ${{ secrets.SLACK_WEBHOOK }} + + deploy: + needs: [ build, build_windows ] + runs-on: ubuntu-18.04 + if: github.event_name == 'push' || github.event_name == 'schedule' + steps: + - uses: actions/checkout@v1 + + - name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + + - name: Build and test and push docker image + env: + IMAGE_NAME: freqtradeorg/freqtrade + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} + run: | + build_helpers/publish_docker.sh + + - name: Build raspberry image for ${{ steps.extract_branch.outputs.branch }}_pi + uses: elgohr/Publish-Docker-Github-Action@2.7 + with: + name: freqtradeorg/freqtrade:${{ steps.extract_branch.outputs.branch }}_pi + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + dockerfile: Dockerfile.pi + # cache: true + cache: ${{ github.event_name != 'schedule' }} + tag_names: true + + - name: Slack Notification + uses: homoluctus/slatify@v1.8.0 + if: always() + with: + type: ${{ job.status }} + job_name: '*Freqtrade CI Deploy*' + mention: 'here' + mention_if: 'failure' + channel: '#notifications' + url: ${{ secrets.SLACK_WEBHOOK }} + diff --git a/.github/workflows/docker_update_readme.yml b/.github/workflows/docker_update_readme.yml index 634517dc9..57a7e591e 100644 --- a/.github/workflows/docker_update_readme.yml +++ b/.github/workflows/docker_update_readme.yml @@ -8,7 +8,7 @@ jobs: dockerHubDescription: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v1 - name: Docker Hub Description uses: peter-evans/dockerhub-description@v2.1.0 env: diff --git a/.travis.yml b/.travis.yml index 8aaff553f..80e4080d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,11 +45,11 @@ jobs: - script: mypy freqtrade scripts name: mypy - - stage: docker - if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron)) - script: - - build_helpers/publish_docker.sh - name: "Build and test and push docker image" + # - stage: docker + # if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron)) + # script: + # - build_helpers/publish_docker.sh + # name: "Build and test and push docker image" notifications: slack: diff --git a/Dockerfile.pi b/Dockerfile.pi index 85ba5892f..279f85a04 100644 --- a/Dockerfile.pi +++ b/Dockerfile.pi @@ -38,3 +38,4 @@ RUN ~/berryconda3/bin/pip install -e . --no-cache-dir RUN [ "cross-build-end" ] ENTRYPOINT ["/root/berryconda3/bin/python","./freqtrade/main.py"] +CMD [ "trade" ] diff --git a/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl new file mode 100644 index 000000000..87469a199 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl differ diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 new file mode 100644 index 000000000..30427c3cc --- /dev/null +++ b/build_helpers/install_windows.ps1 @@ -0,0 +1,8 @@ +# Downloads don't work automatically, since the URL is regenerated via javascript. +# Downloaded from https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib +# Invoke-WebRequest -Uri "https://download.lfd.uci.edu/pythonlibs/xxxxxxx/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" -OutFile "TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" + +pip install build_helpers\TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl + +pip install -r requirements-dev.txt +pip install -e . diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index b8318c196..7fe4b17eb 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -1,17 +1,17 @@ #!/bin/sh -# - export TAG=`if [ "$TRAVIS_BRANCH" == "develop" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` -# Replace / with _ to create a valid tag -TAG=$(echo "${TRAVIS_BRANCH}" | sed -e "s/\//_/") +# Replace / with _ to create a valid tag +TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") +echo "Running for ${TAG}" # Add commit and commit_message to docker container -echo "${TRAVIS_COMMIT} ${TRAVIS_COMMIT_MESSAGE}" > freqtrade_commit +echo "${GITHUB_SHA}" > freqtrade_commit -if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then - echo "event ${TRAVIS_EVENT_TYPE}: full rebuild - skipping cache" +if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then + echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" docker build -t freqtrade:${TAG} . else - echo "event ${TRAVIS_EVENT_TYPE}: building with cache" + echo "event ${GITHUB_EVENT_NAME}: building with cache" # Pull last build to avoid rebuilding the whole image docker pull ${IMAGE_NAME}:${TAG} docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . @@ -23,7 +23,7 @@ if [ $? -ne 0 ]; then fi # Run backtest -docker run --rm -it -v $(pwd)/config.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy DefaultStrategy +docker run --rm -v $(pwd)/config.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy DefaultStrategy if [ $? -ne 0 ]; then echo "failed running backtest" @@ -38,12 +38,12 @@ if [ $? -ne 0 ]; then fi # Tag as latest for develop builds -if [ "${TRAVIS_BRANCH}" = "develop" ]; then +if [ "${GITHUB_REF}" = "develop" ]; then docker tag freqtrade:$TAG ${IMAGE_NAME}:latest fi # Login -echo "$DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin +docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD if [ $? -ne 0 ]; then echo "failed login" diff --git a/docs/developer.md b/docs/developer.md index 67c0912a7..41c66ce37 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -198,6 +198,19 @@ jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace user_data/not jupyter nbconvert --ClearOutputPreprocessor.enabled=True --to markdown user_data/notebooks/strategy_analysis_example.ipynb --stdout > docs/strategy_analysis_example.md ``` +## Continuous integration + +This documents some decisions taken for the CI Pipeline. + +* CI runs on all OS variants, Linux (ubuntu), macOS and Windows. +* Docker images are build for the branches `master` and `develop`. +* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:master_pi` and `develop_pi`. +* Docker images contain a file, `/freqtrade/freqtrade_commit` containing the commit this image is based of. +* Full docker image rebuilds are run once a week via schedule. +* Deployments run on ubuntu. +* ta-lib binaries are contained in the build_helpers directory to avoid fails related to external unavailability. +* All tests must pass for a PR to be merged to `master` or `develop`. + ## Creating a release This part of the documentation is aimed at maintainers, and shows how to create a release. diff --git a/tests/test_docs.sh b/tests/test_docs.sh new file mode 100755 index 000000000..09e142b99 --- /dev/null +++ b/tests/test_docs.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Test Documentation boxes - +# !!! : is not allowed! +# !!! "title" - Title needs to be quoted! +grep -Er '^!{3}\s\S+:|^!{3}\s\S+\s[^"]' docs/* + +if [ $? -ne 0 ]; then + echo "Docs test success." + exit 0 +fi +echo "Docs test failed." +exit 1