Merge pull request #220 from AXErunners/development

Release 1.4.1

.gitignore

@@ -83,6 +83,7 @@ libconftest.dylib*
 Makefile
 axe-qt
 Axe-Qt.app
+!/depends/Makefile
 
 # Unit-tests
 Makefile.test
@@ -108,9 +109,9 @@ coverage_percent.txt
 linux-coverage-build
 linux-build
 win32-build
-qa/pull-tester/tests_config.py
-qa/pull-tester/tests_config.ini
-qa/cache/*
+test/functional/config.ini
+test/util/buildenv.py
+test/cache/*
 
 !src/leveldb*/Makefile
 

.gitlab-ci.yml

@@ -0,0 +1,144 @@
+image: "ubuntu:bionic"
+
+variables:
+ DOCKER_DRIVER: overlay2
+
+cache:
+ # Cache by branch/tag and job name
+ # Gitlab can't use caches from parent pipelines when doing the first build in a PR, so we use artifacts to copy
+ # caches into PRs
+ key: ${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}${CI_EXTERNAL_PULL_REQUEST_IID}
+ paths:
+ - $CI_PROJECT_DIR/cache
+
+stages:
+ - build
+
+.build_template: &build_template
+ stage: build
+ before_script:
+ - export BUILD_TARGET="$CI_JOB_NAME"
+ - echo BUILD_TARGET=$BUILD_TARGET
+ - source ./ci/matrix.sh
+
+ # The ubuntu base image has apt configured to delete caches after each invocation, which is something that is not desirable for us
+ - rm /etc/apt/apt.conf.d/docker-clean
+ - apt-get update
+ - apt-get install -y wget unzip
+
+ # Init cache
+ - export CACHE_DIR=$CI_PROJECT_DIR/cache
+ - mkdir -p $CACHE_DIR
+ - |
+ if [ "$CI_COMMIT_REF_SLUG" != "development" -a "$CI_COMMIT_TAG" == "" ]; then
+ if [ ! -d $CACHE_DIR/ccache ]; then
+ echo "Downloading cache from development branch"
+ mkdir cache-artifact
+ cd cache-artifact
+ if wget --quiet -O cache-artifact.zip https://gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/-/jobs/artifacts/development/download?job=$CI_JOB_NAME; then
+ unzip -q cache-artifact.zip
+ rm cache-artifact.zip
+ mv cache-artifact/* $CACHE_DIR/
+ else
+ echo "Failed to download cache"
+ fi
+ cd ..
+ rm -rf cache-artifact
+ else
+ echo "Not touching cache (was initialized from previous build)"
+ fi
+ else
+ echo "Not touching cache (building development branch or tag)"
+ fi
+ # Create missing cache dirs
+ - mkdir -p $CACHE_DIR/ccache && mkdir -p $CACHE_DIR/depends && mkdir -p $CACHE_DIR/sdk-sources && mkdir -p $CACHE_DIR/apt
+ # Keep this as it makes caching related debugging easier
+ - ls -lah $CACHE_DIR && ls -lah $CACHE_DIR/depends && ls -lah $CACHE_DIR/ccache && ls -lah $CACHE_DIR/apt
+ - mv $CACHE_DIR/apt/* /var/cache/apt/archives/ || true
+
+ # Install base packages
+ - apt-get dist-upgrade -y
+ - apt-get install -y git g++ autotools-dev libtool m4 automake autoconf pkg-config zlib1g-dev libssl1.0-dev curl ccache bsdmainutils cmake
+ - apt-get install -y python3 python3-dev python3-pip
+
+ # jinja2 is needed for combine_logs.py
+ - pip3 install jinja2
+
+ # Setup some environment variables
+ - if [ "$CI_EXTERNAL_PULL_REQUEST_IID" != "" ]; then export PULL_REQUEST="true"; else export PULL_REQUEST="false"; fi
+ - export COMMIT_RANGE="$CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA"
+ - export JOB_NUMBER="$CI_JOB_ID"
+ - export HOST_SRC_DIR=$CI_PROJECT_DIR
+ - echo PULL_REQUEST=$PULL_REQUEST COMMIT_RANGE=$COMMIT_RANGE HOST_SRC_DIR=$HOST_SRC_DIR CACHE_DIR=$CACHE_DIR
+ - echo "Commit log:" && git log --format=fuller -1
+
+ # Build axe_hash
+ - git clone https://github.com/axerunners/axe_hash
+ - cd axe_hash && python3 setup.py install
+
+ # Install build target specific packages
+ - echo PACKAGES=$PACKAGES
+ - if [ -n "$DPKG_ADD_ARCH" ]; then dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi
+ - if [ -n "$PACKAGES" ]; then apt-get update && apt-get install -y --no-install-recommends --no-upgrade $PACKAGES; fi
+
+ # Move apt packages into cache
+ - mv /var/cache/apt/archives/* $CACHE_DIR/apt/ || true
+
+ # Make mingw use correct threading libraries
+ - update-alternatives --set i686-w64-mingw32-gcc /usr/bin/i686-w64-mingw32-gcc-posix || true
+ - update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix || true
+ - update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix || true
+ - update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix || true
+
+ script:
+ - export BUILD_TARGET="$CI_JOB_NAME"
+ - cd $CI_PROJECT_DIR
+ - ./ci/build_depends.sh
+ - ./ci/build_src.sh
+ - ./ci/test_unittests.sh
+ - ./ci/test_integrationtests.sh
+
+ after_script:
+ # Copy all cache files into cache-artifact so that they get uploaded. We only do this for development so that artifacts
+ # stay minimal for PRs and branches (we never need them)
+ - mkdir -p $CI_PROJECT_DIR/cache-artifact
+ - mkdir -p $CI_PROJECT_DIR/testlogs
+ - |
+ if [ "$CI_COMMIT_REF_SLUG" = "development" ]; then
+ cp -ra $CI_PROJECT_DIR/cache/* $CI_PROJECT_DIR/cache-artifact/
+ fi
+
+ # We're actually only interested in the development branch creating the cache artifact, but there is no way to control this
+ # until https://gitlab.com/gitlab-org/gitlab-foss/issues/25478 gets implemented. Until then, we use an expiration time of
+ # 3 days and rely on daily builds to refresh the cache artifacts. We also keep non-development artifacts at minimum size
+ artifacts:
+ name: cache-artifact
+ when: always
+ paths:
+ - $CI_PROJECT_DIR/cache-artifact
+ - $CI_PROJECT_DIR/testlogs
+ expire_in: 3 days
+
+arm-linux:
+ <<: *build_template
+
+win32:
+ <<: *build_template
+
+win64:
+ <<: *build_template
+
+linux32:
+ <<: *build_template
+
+linux64:
+ <<: *build_template
+
+linux64_nowallet:
+ <<: *build_template
+
+linux64_release:
+ <<: *build_template
+
+mac:
+ <<: *build_template

.travis.yml

@@ -6,21 +6,12 @@ sudo: required
 dist: trusty
 
 os: linux
-language: generic
-
-addons:
- apt:
- packages:
- # Use more recent docker version
- - docker-ce
- # Can be removed if Travis ever upgrades to Bionic
- - realpath
+language: minimal
 
 services:
  - docker
 
 cache:
- apt: true
  ccache: true
  directories:
  - $HOME/cache
@@ -31,26 +22,108 @@ env:
  - secure: "xdeQldckRR0Rp+GJr5BV0NRFaR0dGX4469LqAXP8Gn+01ZXtsVuwsereN177nbtRtGcCZYRukrVe9LWvksgWFoHfKuSv82J8TxFRIkXeXbAN1vLJpgQ+MI+iGOt940aiuy8P3eubbHAhqk9kCa9CuQMxtierG7DwgH2ri1zal5Q3x3wQtp0zvq8l36uAma0sGBK9Vz/N+Y+rYc/hAMKUMN8iZ0lt87Vili9hZmpnO7FJ2kkCcD3A24TrD9xnspWxbGO7F6rfWvc6RzbB9X6xQtuzoor7DHPlih93mNmBHjfv5KOwzc8oHY7ry2/54/yGXX9IvANgY3iW24YnyVBCVWaCpRfBG/cgBFbxjfcmaOK2nZHFuaPfpDYc346thQcwu3srPeDJjBaCXaYprrPZWnvaP0OpUcY4M+rxcNcAUY1BmDTa7Fdc1RwbBWJSsdc6qL2XxMR9cZreOYBNN+BWTcZDnnlZ+3zkGcR+P4WUYTvODdqNNj+mm+VwcYxoZzQvtOFyAtsgcEa+47UqFuhczMhU5CocX1fRMdjO0RWWG3A3CenPOMifvyPDSaHe+oPGTpxJTpXw3V5Ceu0gwkSTmMkAx1ZT1qhf7jxcfl9rieCNqF8iQHXOjkiI2mjpI2cK2Ld5IR6V3FNZ2/2MHTs1uGHwVYVoDMUbFCwP2Tcbbes="
  - DOCKER_BUILD=false
 
- matrix:
- - BUILD_TARGET=arm-linux
- - BUILD_TARGET=win32
- - BUILD_TARGET=win64
- - BUILD_TARGET=linux32
- - BUILD_TARGET=linux64
- - BUILD_TARGET=linux64_nowallet
- - BUILD_TARGET=linux64_release DOCKER_BUILD=true
- - BUILD_TARGET=mac
+stages:
+ - build depends
+ - build src
+ - run tests
+ - build docker
+
+builddepends: &builddepends
+ stage: build depends
+ script:
+ - $DOCKER_RUN_IN_BUILDER ./ci/build_depends.sh
+
+buildsrc: &buildsrc
+ stage: build src
+ script:
+ - $DOCKER_RUN_IN_BUILDER ./ci/build_depends.sh
+ - $DOCKER_RUN_IN_BUILDER ./ci/build_src.sh
+
+runtests: &runtests
+ stage: run tests
+ script:
+ - $DOCKER_RUN_IN_BUILDER ./ci/build_depends.sh
+ - $DOCKER_RUN_IN_BUILDER ./ci/build_src.sh
+ - $DOCKER_RUN_IN_BUILDER ./ci/test_unittests.sh
+ - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then extended="--extended --exclude pruning,dbcrash"; fi
+ - $DOCKER_RUN_IN_BUILDER ./ci/test_integrationtests.sh --jobs=3 ${extended}
+
+builddocker: &builddocker
+ stage: build docker
+ script:
+ # no need to run tests again here
+ - if [ "$DOCKER_BUILD" = "true" ]; then $DOCKER_RUN_IN_BUILDER ./ci/build_depends.sh && $DOCKER_RUN_IN_BUILDER ./ci/build_src.sh && BUILD_DIR=build-ci/axecore-$BUILD_TARGET ./docker/build-docker.sh; fi
+
+jobs:
+ include:
+ # build depends
+ - <<: *builddepends
+ env: BUILD_TARGET=arm-linux
+ - <<: *builddepends
+ env: BUILD_TARGET=win32
+ - <<: *builddepends
+ env: BUILD_TARGET=win64
+ - <<: *builddepends
+ env: BUILD_TARGET=linux32
+ - <<: *builddepends
+ env: BUILD_TARGET=linux64
+ - <<: *builddepends
+ env: BUILD_TARGET=linux64_nowallet
+ - <<: *builddepends
+ env: BUILD_TARGET=linux64_release DOCKER_BUILD=true
+ - <<: *builddepends
+ env: BUILD_TARGET=mac
+ # build source
+ - <<: *buildsrc
+ env: BUILD_TARGET=arm-linux
+ - <<: *buildsrc
+ env: BUILD_TARGET=win32
+ - <<: *buildsrc
+ env: BUILD_TARGET=win64
+ - <<: *buildsrc
+ env: BUILD_TARGET=linux32
+ - <<: *buildsrc
+ env: BUILD_TARGET=linux64
+ - <<: *buildsrc
+ env: BUILD_TARGET=linux64_nowallet
+ - <<: *buildsrc
+ env: BUILD_TARGET=linux64_release DOCKER_BUILD=true
+ - <<: *buildsrc
+ env: BUILD_TARGET=mac
+ # run tests (no tests for arm-linux and mac)
+ - <<: *runtests
+ env: BUILD_TARGET=win32
+ - <<: *runtests
+ env: BUILD_TARGET=win64
+ - <<: *runtests
+ env: BUILD_TARGET=linux32
+ - <<: *runtests
+ env: BUILD_TARGET=linux64
+ - <<: *runtests
+ env: BUILD_TARGET=linux64_nowallet
+ - <<: *runtests
+ env: BUILD_TARGET=linux64_release DOCKER_BUILD=true
+ # build docker
+ - <<: *builddocker
+ env: BUILD_TARGET=linux64_release DOCKER_BUILD=true
 
 before_cache:
  # Save builder image
  - docker save axe-builder-$BUILD_TARGET-$TRAVIS_JOB_NUMBER $(docker history -q axe-builder-$BUILD_TARGET-$TRAVIS_JOB_NUMBER | grep -v \<missing\>) | gzip -2 > $HOME/cache/docker/axe-builder-$BUILD_TARGET.tar.gz
 
+before_install:
+ - travis_retry travis_apt_get_update
+ - travis_retry sudo apt-get -yq --no-install-suggests --no-install-recommends install docker-ce realpath
+
 install:
  # Our scripts try to be Travis agnostic
  - export PULL_REQUEST="$TRAVIS_PULL_REQUEST"
+ - export COMMIT_RANGE="$TRAVIS_COMMIT_RANGE"
  - export JOB_NUMBER="$TRAVIS_JOB_NUMBER"
  - export HOST_SRC_DIR=$TRAVIS_BUILD_DIR
  - export HOST_CACHE_DIR=$HOME/cache
+ - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1`
+ - export PYTHON_DEBUG=1
  - source ./ci/matrix.sh
  - mkdir -p $HOST_CACHE_DIR/docker && mkdir -p $HOST_CACHE_DIR/ccache && mkdir -p $HOST_CACHE_DIR/depends && mkdir -p $HOST_CACHE_DIR/sdk-sources
  # Keep this as it makes caching related debugging easier
@@ -65,18 +138,6 @@ before_script:
  - python3 -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
  # Build docker image only for development branch of the main repo
  - if [ "$TRAVIS_REPO_SLUG" != "axerunners/axe" -o "$TRAVIS_BRANCH" != "development" -o "$TRAVIS_PULL_REQUEST" != "false" ]; then export DOCKER_BUILD="false"; echo DOCKER_BUILD=$DOCKER_BUILD; fi
-script:
- - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1`
- # Our scripts try to be Travis agnostic
- - $DOCKER_RUN_IN_BUILDER ./ci/build_depends.sh;
- # Gracefully stop build without running into timeouts (which won't update caches) when building depends or source took too long
- # Next build should fix this situation as it will start with a populated cache
- - if [ $SECONDS -gt 1200 ]; then export TIMEOUT="true"; false; fi # The "false" here ensures that the build is marked as failed even though the whole script returns 0
- - test "$TIMEOUT" != "true" && $DOCKER_RUN_IN_BUILDER ./ci/build_src.sh
- - test "$TIMEOUT" != "true" && $DOCKER_RUN_IN_BUILDER ./ci/test_unittests.sh
- - if [ $SECONDS -gt 1800 -a "$RUN_INTEGRATIONTESTS" = "true" ]; then export TIMEOUT="true"; false; fi # The "false" here ensures that the build is marked as failed even though the whole script returns 0
- - test "$TIMEOUT" != "true" && $DOCKER_RUN_IN_BUILDER ./ci/test_integrationtests.sh --jobs=3
- - test "$TIMEOUT" != "true" && if [ "$DOCKER_BUILD" = "true" ]; then BUILD_DIR=build-ci/axecore-$BUILD_TARGET ./docker/build-docker.sh; fi
 after_script:
  - echo $TRAVIS_COMMIT_RANGE
  - echo $TRAVIS_COMMIT_LOG

INSTALL.md

@@ -2,4 +2,4 @@ Building AXE
 =============
 
 See doc/build-*.md for instructions on building the various
-elements of the AXE Core reference implementation of AXE.
+elements of the Axe Core reference implementation of AXE.