Refactor docker image publishing

This adds support to push docker images to quay.io, like other projects in
the iovisor org.

It separates docker image builds into a separate github workflow, and
refactors the package building process slightly, to be generic, in order to
create builds for both ubuntu 16.04 and ubuntu 18.04.

This provides a means to distribute intermediate apt packages between releases,
and also enables uploading these as CI artifacts.

As recent releases have not annotated their tags, it drops the requirement for
tags to be annotated in selecting the version to use.

.github/workflows/bcc-test.yml

@@ -92,38 +92,3 @@ jobs:
 # https://github.com/marketplace/actions/debugging-with-tmate
 # - name: Setup tmate session
 # uses: mxschmitt/action-tmate@v1
-
- # Optionally publish container images, guarded by the GitHub secret
- # DOCKER_PUBLISH.
- # GitHub secrets can be configured as follows:
- # - DOCKER_PUBLISH = 1
- # - DOCKER_IMAGE = docker.io/myorg/bcc
- # - DOCKER_USERNAME = username
- # - DOCKER_PASSWORD = password
- publish:
- name: Publish
- runs-on: ubuntu-latest
- steps:
-
- - uses: actions/checkout@v1
-
- - name: Initialize workflow variables
- id: vars
- shell: bash
- run: |
- echo ::set-output name=DOCKER_PUBLISH::${DOCKER_PUBLISH}
- env:
- DOCKER_PUBLISH: "${{ secrets.DOCKER_PUBLISH }}"
-
- - name: Build container image and publish to registry
- id: publish-registry
- uses: elgohr/[email protected]
- if: ${{ steps.vars.outputs.DOCKER_PUBLISH }}
- with:
- name: ${{ secrets.DOCKER_IMAGE }}
- username: ${{ secrets.DOCKER_USERNAME }}
- password: ${{ secrets.DOCKER_PASSWORD }}
- workdir: .
- dockerfile: Dockerfile.ubuntu
- snapshot: true
- cache: ${{ github.event_name != 'schedule' }}

.github/workflows/publish.yml

@@ -0,0 +1,101 @@
+name: Publish Build Artifacts
+
+on: push
+
+jobs:
+ publish_images:
+ # Optionally publish container images, guarded by the GitHub secret
+ # QUAY_PUBLISH.
+ # To set this up, sign up for quay.io (you can connect it to your github)
+ # then create a robot user with write access user called "bcc_buildbot",
+ # and add the secret token for it to GitHub secrets as:
+ # - QUAY_TOKEN = <token from quay.io>
+ name: Publish to quay.io
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ env:
+ - NAME: xenial-release
+ OS_RELEASE: 16.04
+ - NAME: bionic-release
+ OS_RELEASE: 18.04
+ steps:
+
+ - uses: actions/checkout@v1
+
+ - name: Initialize workflow variables
+ id: vars
+ shell: bash
+ run: |
+ if [ -n "${QUAY_TOKEN}" ];then
+ echo "Quay token is set, will push an image"
+ echo ::set-output name=QUAY_PUBLISH::true
+ else
+ echo "Quay token not set, skipping"
+ fi
+
+ env:
+ QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
+
+ - name: Authenticate with quay.io docker registry
+ if: >
+ steps.vars.outputs.QUAY_PUBLISH
+ env:
+ QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
+ run: ./scripts/docker/auth.sh ${{ github.repository }}
+
+ - name: Package docker image and push to quay.io
+ if: >
+ steps.vars.outputs.QUAY_PUBLISH
+ run: >
+ ./scripts/docker/push.sh
+ ${{ github.repository }}
+ ${{ github.ref }}
+ ${{ github.sha }}
+ ${{ matrix.env['NAME'] }}
+ ${{ matrix.env['OS_RELEASE'] }}
+
+ # Uploads the packages built in docker to the github build as an artifact for convenience
+ - uses: actions/upload-artifact@v1
+ with:
+ name: ${{ matrix.env['NAME'] }}
+ path: output
+
+ # Optionally publish container images to custom docker repository,
+ # guarded by presence of all required github secrets.
+ # GitHub secrets can be configured as follows:
+ # - DOCKER_IMAGE = docker.io/myorg/bcc
+ # - DOCKER_USERNAME = username
+ # - DOCKER_PASSWORD = password
+ publish_dockerhub:
+ name: Publish To Dockerhub
+ runs-on: ubuntu-latest
+ steps:
+
+ - uses: actions/checkout@v1
+
+ - name: Initialize workflow variables
+ id: vars
+ shell: bash
+ run: |
+ if [ -n "${DOCKER_IMAGE}" ] && \
+ [ -n "${DOCKER_USERNAME}" ] && \
+ [ -n "${DOCKER_PASSWORD}" ];then
+ echo "Custom docker credentials set, will push an image"
+ echo ::set-output name=DOCKER_PUBLISH::true
+ else
+ echo "Custom docker credentials not, skipping"
+ fi
+
+ - name: Build container image and publish to registry
+ id: publish-registry
+ uses: elgohr/[email protected]
+ if: ${{ steps.vars.outputs.DOCKER_PUBLISH }}
+ with:
+ name: ${{ secrets.DOCKER_IMAGE }}
+ username: ${{ secrets.DOCKER_USERNAME }}
+ password: ${{ secrets.DOCKER_PASSWORD }}
+ workdir: .
+ dockerfile: Dockerfile.ubuntu
+ snapshot: true
+ cache: ${{ github.event_name != 'schedule' }}

.gitignore

@@ -12,3 +12,7 @@ debian/**/*.log
 *critical.log
 obj-x86_64-linux-gnu
 examples/cgroupid/cgroupid
+
+# Output from docker builds
+scripts/docker/output/
+/output/

Dockerfile.ubuntu

@@ -1,4 +1,9 @@
-FROM ubuntu:bionic as builder
+ARG OS_TAG=18.04
+FROM ubuntu:${OS_TAG} as builder
+
+ARG OS_TAG
+ARG BUILD_TYPE=release
+ARG DEBIAN_FRONTEND=noninteractive
 
 MAINTAINER Brenden Blanco <[email protected]>
 
@@ -10,10 +15,9 @@ COPY ./ /root/bcc
 WORKDIR /root/bcc
 
 RUN /usr/lib/pbuilder/pbuilder-satisfydepends && \
- ./scripts/build-deb.sh
-
+ ./scripts/build-deb.sh ${BUILD_TYPE}
 
-FROM ubuntu:bionic
+FROM ubuntu:${OS_TAG}
 
 COPY --from=builder /root/bcc/*.deb /root/bcc/
 

scripts/build-deb.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# helper script to be invoked by jenkins/buildbot
+# helper script to be invoked by jenkins/buildbot or github actions
 
 # $1 [optional]: the build type - release | nightly | test
 buildtype=${1:-test}

scripts/docker/auth.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+# For now only quay.io is supported, but this could be portable to dockerhub
+# and other image repositories.
+
+# Forks can push using this approach if they create a quay.io bot user
+# with name matching of ORGNAME+bcc_buildbot, or by setting QUAY_BOT_NAME
+
+git_repo=$1 # github.repository format: ORGNAME/REPONAME
+
+# Set this value as QUAY_TOKEN in the github repository settings "Secrets" tab
+[[ -z "${QUAY_TOKEN}" ]] && echo "QUAY_TOKEN not set" && exit 0
+
+# Set this to match the name of the bot user on quay.io
+[[ -z "${QUAY_BOT_NAME}" ]] && QUAY_BOT_NAME="bcc_buildbot"
+
+quay_user="$(dirname ${git_repo})+${QUAY_BOT_NAME}"
+echo "${QUAY_TOKEN}" | docker login -u="${quay_user}" --password-stdin quay.io

scripts/docker/build.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Builds debian packages using docker wrapper
+
+function help() {
+ message=$1
+ echo "USAGE: build.sh DOCKER_REPO DOCKER_TAG OS_TAG [DISTRO]"
+ echo "hint: ${message}"
+}
+
+docker_repo=$1
+docker_tag=$2
+os_tag=$3
+distro=${4:-ubuntu}
+
+[ -z "${docker_repo}" ] && help "You must specify repo, eg: quay.io/iovisoc/bcc" && exit 1
+[ -z "${docker_tag}" ] && help "You must specify tag, eg: bionic-release-master, latest, SHA, git tag, etc " && exit 1
+[ -z "${os_tag}" ] && help "You must specify os tag, eg: 18.04, bionic, etc " && exit 1
+
+
+# The main docker image build,
+echo "Building ${distro} ${os_tag} release docker image for ${docker_repo}:${docker_tag}"
+docker build -t ${docker_repo}:${docker_tag} --build-arg OS_TAG=${os_tag} -f Dockerfile.${distro} .
+
+echo "Copying build artifacts to $(pwd)/output"
+mkdir output
+docker run -v $(pwd)/output:/output ${docker_repo}:${docker_tag} /bin/bash -c "cp /root/bcc/* /output"

scripts/docker/push.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+# Push docker tags to a configured docker repo, defaulting to quay.io
+# You must run login.sh before running this script.
+
+DEFAULT_DOCKER_REPO="quay.io"
+DEFAULT_RELEASE_TARGET="bionic-release" # will allow unprefixed tags
+
+# Currently only support pushing to quay.io
+DOCKER_REPO=${DEFAULT_DOCKER_REPO}
+
+git_repo=$1 # github.repository format: ORGNAME/REPONAME
+git_ref=$2 # github.ref format: refs/REMOTE/REF
+ # eg, refs/heads/BRANCH
+ # refs/tags/v0.9.6-pre
+git_sha=$3 # github.sha GIT_SHA
+type_name=$4 # build name, s/+/_/g eg, bionic-release
+os_tag=${5:-18.04} # numeric docker tag eg, 18.04
+
+# refname will be either a branch like "master" or "some-branch",
+# or a tag, like "v1.17.0-pre".
+# When a tag is pushed, a build is done for both the branch and the tag, as
+# separate builds.
+# This is a feature specific to github actions based on the `github.ref` object
+refname=$(basename ${git_ref})
+
+# The build type needs to be sanitized into a valid tag, replacing + with _
+type_tag="$(echo ${type_name} | sed 's/+/_/g')"
+
+
+echo "Triggering image build"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+${SCRIPT_DIR}/build.sh ${DOCKER_REPO}/${git_repo} ${git_sha}-${type_tag} ${os_tag}
+
+echo "Upload image for git sha ${git_sha} to ${DOCKER_REPO}/${git_repo}"
+docker push ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag}
+
+echo "Push tags to branch or git tag HEAD refs"
+docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:${refname}-${type_tag}
+docker push ${DOCKER_REPO}/${git_repo}:${refname}-${type_tag}
+
+# Only push to un-suffixed tags for the default release target build type
+if [[ "${type_name}" == "${DEFAULT_RELEASE_TARGET}"* ]];then
+
+ # Update branch / git tag ref
+ echo "Pushing tags for ${DOCKER_REPO}/${git_repo}:${refname}"
+ docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:${refname}
+ docker push ${DOCKER_REPO}/${git_repo}:${refname}
+
+ if [[ "${refname}" == "master" ]];then
+ if [[ "${edge}" == "ON" ]];then
+ echo "This is an edge build on master, pushing ${DOCKER_REPO}/${git_repo}:edge"
+ docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:edge
+ docker push ${DOCKER_REPO}/${git_repo}:edge
+ else
+ echo "This is a build on master, pushing ${DOCKER_REPO}/${git_repo}:latest :SHA as well"
+ docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:latest
+ docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:${git_sha}
+ docker push ${DOCKER_REPO}/${git_repo}:latest
+ docker push ${DOCKER_REPO}/${git_repo}:${git_sha}
+ fi
+ fi
+fi

scripts/git-tag.sh

@@ -1,4 +1,4 @@
-git_tag_latest=$(git describe --abbrev=0)
+git_tag_latest=$(git describe --tags --abbrev=0)
 git_rev_count=$(git rev-list $git_tag_latest.. --count)
 git_rev_count=$[$git_rev_count+1]
 git_subject=$(git log --pretty="%s" -n 1)