Skip to content

Commit

Permalink
Initial deb and rpm packaging (#405)
Browse files Browse the repository at this point in the history
* wip

* Initial deb and rpm packaging

- Uses https://github.com/jordansissel/fpm
- Adds `deb-package` and `rpm-package` targets to Makefile
- Builds simple deb and rpm packages that installs the otelcontribcol
binary to `/usr/bin/otelcontribcol`
- Adds deb and rpm packages to circleci workspace for github release
- TODO: create service scripts

* Packaging updates

- Add support for arm64 packages
- Move files to internal/buildscripts/packaging/
- Use single Dockerfile for local deb and rpm builds
- Explicitly set output dir for packages in circleci

* Add systemd service to deb and rpm

- Install `otel-contrib-collector` systemd service and run as `otel:otel` user:group
- Add simple installation test scripts using systemd-in-docker images

* Change default output dir to dist

Co-authored-by: Owais Lone <[email protected]>
  • Loading branch information
jchengsfx and owais committed Jul 15, 2020
1 parent 6a25486 commit d1598f3
Show file tree
Hide file tree
Showing 18 changed files with 481 additions and 7 deletions.
119 changes: 116 additions & 3 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,33 @@ executors:
image: ubuntu-1604:201903-01

commands:
verify_dist_files_exist:
parameters:
files:
type: string
default: |
bin/otelcontribcol_darwin_amd64
bin/otelcontribcol_linux_arm64
bin/otelcontribcol_linux_amd64
bin/otelcontribcol_windows_amd64.exe
dist/otel-contrib-collector-*.arm64.rpm
dist/otel-contrib-collector_*_amd64.deb
dist/otel-contrib-collector-*.x86_64.rpm
dist/otel-contrib-collector_*_arm64.deb
dist/opentelemetry-contrib-collector.msi
steps:
- run:
name: Check if files exist
command: |
files="<< parameters.files >>"
for f in $files; do
if [[ ! -f $f ]]
then
echo "$f does not exist."
exit 1
fi
done
setup:
steps:
- checkout
Expand Down Expand Up @@ -160,13 +187,24 @@ workflows:
filters:
tags:
only: /.*/
- publish-check:
requires:
- lint
- unit-tests
- integration-tests
- cross-compile
- windows-msi
- deb-package
- rpm-package
- publish-stable:
requires:
- lint
- unit-tests
- integration-tests
- cross-compile
- windows-msi
- deb-package
- rpm-package
filters:
branches:
ignore: /.*/
Expand All @@ -187,6 +225,22 @@ workflows:
filters:
tags:
only: /.*/
- build-package:
name: deb-package
package_type: deb
requires:
- cross-compile
filters:
tags:
only: /^v([0-9])+.([0-9])+.([0-9])+.*/
- build-package:
name: rpm-package
package_type: rpm
requires:
- cross-compile
filters:
tags:
only: /^v([0-9])+.([0-9])+.([0-9])+.*/

jobs:
setup:
Expand Down Expand Up @@ -261,6 +315,8 @@ jobs:
shell: powershell.exe
steps:
- attach_to_workspace
- run:
command: mkdir -p dist
- run:
name: Install Wix Toolset
command: .\internal\buildscripts\packaging\msi\make.ps1 Install-Tools
Expand All @@ -274,28 +330,51 @@ jobs:
command: .\internal\buildscripts\packaging\msi\make.ps1 Confirm-MSI
- persist_to_workspace:
root: ~/
paths: project/bin/*.msi
paths: project/dist/*.msi

publish-check:
docker:
- image: cimg/go:1.14
steps:
- attach_to_workspace
- setup_remote_docker
- verify_dist_files_exist
- run:
name: Check passed
command: echo "publish check passed meaning release CI jobs should work as expected"
when: on_success
- run:
name: Check failed
command: echo "publish check failed. This means release CI jobs will likely fail as well"
when: on_fail

publish-stable:
docker:
- image: cimg/go:1.14
steps:
- restore_workspace
- verify_dist_files_exist
- setup_remote_docker
- publish_docker_images:
repo: opentelemetry-collector-contrib
tag: ${CIRCLE_TAG:1}
- run:
name: Prepare release artifacts
command: |
cp bin/* dist/
mv dist/opentelemetry-contrib-collector.msi dist/otel-contrib-collector.msi
- run:
name: Calculate checksums
command: cd bin && shasum -a 256 * > checksums.txt
command: cd dist && shasum -a 256 * > checksums.txt
- run:
name: Create Github release and upload artifacts
command: ghr -t $GITHUB_TOKEN -u $CIRCLE_PROJECT_USERNAME -r $CIRCLE_PROJECT_REPONAME --replace $CIRCLE_TAG bin/
command: ghr -t $GITHUB_TOKEN -u $CIRCLE_PROJECT_USERNAME -r $CIRCLE_PROJECT_REPONAME --replace $CIRCLE_TAG dist/

publish-dev:
executor: golang
steps:
- restore_workspace
- verify_dist_files_exist
- setup_remote_docker
- publish_docker_images:
repo: opentelemetry-collector-contrib-dev
Expand Down Expand Up @@ -383,3 +462,37 @@ jobs:
path: test-results/junit
- store_artifacts:
path: test-results

build-package:
machine:
image: ubuntu-1604:202007-01
parameters:
package_type:
type: enum
enum: ["deb", "rpm"]
steps:
- checkout
- attach_to_workspace
- run:
name: Install fpm and dependencies
command: |
sudo apt-get update
sudo apt-get install -y ruby ruby-dev rubygems build-essential rpm
gem install --no-document fpm -v 1.11.0
- run:
name: Build << parameters.package_type >> amd64 package
command: ./internal/buildscripts/packaging/fpm/<< parameters.package_type >>/build.sh "${CIRCLE_TAG:-}" "amd64" "./dist/"
- run:
name: Build << parameters.package_type >> arm64 package
command: ./internal/buildscripts/packaging/fpm/<< parameters.package_type >>/build.sh "${CIRCLE_TAG:-}" "arm64" "./dist/"
- run:
name: Test << parameters.package_type >> package installation
command: |
if [[ "<< parameters.package_type >>" = "deb" ]]; then
./internal/buildscripts/packaging/fpm/test.sh dist/otel-contrib-collector*amd64.deb examples/tracing/otel-collector-config.yml
else
./internal/buildscripts/packaging/fpm/test.sh dist/otel-contrib-collector*x86_64.rpm examples/tracing/otel-collector-config.yml
fi
- persist_to_workspace:
root: ~/
paths: project/dist/*.<< parameters.package_type >>
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ local/
*.so
*.dylib
bin/
dist/

# Emacs
*~
Expand Down
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ The OpenTelemetry Collector Contrib contains everything in the [opentelemetry-co

- `awsxray` exporter: Use `peer.service` as segment name when set. (#385)
- `splunk` exporter: Add trace exports support (#359, #399)
- Build and publish Windows MSI (#408)
- Build and publish Windows MSI (#408) and DEB/RPM Linux packages (#405)

## 🧰 Bug fixes 🧰

Expand Down
7 changes: 7 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -176,3 +176,10 @@ otel-from-lib:
build-examples:
docker-compose -f examples/tracing/docker-compose.yml build
docker-compose -f exporter/splunkhecexporter/example/docker-compose.yml build

.PHONY: deb-rpm-package
%-package: ARCH ?= amd64
%-package:
$(MAKE) otelcontribcol-linux_$(ARCH)
docker build -t otelcontribcol-fpm internal/buildscripts/packaging/fpm
docker run --rm -v $(CURDIR):/repo -e PACKAGE=$* -e VERSION=$(VERSION) -e ARCH=$(ARCH) otelcontribcol-fpm
16 changes: 16 additions & 0 deletions internal/buildscripts/packaging/fpm/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
FROM debian:9

RUN apt-get update && \
apt-get install -y ruby ruby-dev rubygems build-essential git rpm

RUN gem install --no-document fpm -v 1.11.0

VOLUME /repo
WORKDIR /repo

ENV PACKAGE="deb"
ENV VERSION=""
ENV ARCH="amd64"
ENV OUTPUT_DIR="/repo/dist/"

CMD ./internal/buildscripts/packaging/fpm/$PACKAGE/build.sh "$VERSION" "$ARCH" "$OUTPUT_DIR"
58 changes: 58 additions & 0 deletions internal/buildscripts/packaging/fpm/common.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/bin/bash

FPM_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )"

PKG_NAME="otel-contrib-collector"
PKG_VENDOR="OpenTelemetry Community"
PKG_MAINTAINER="OpenTelemetry Community <[email protected]>"
PKG_DESCRIPTION="OpenTelemetry Contrib Collector"
PKG_LICENSE="Apache 2.0"
PKG_URL="https://github.com/open-telemetry/opentelemetry-collector-contrib"
PKG_USER="otel"
PKG_GROUP="otel"

SERVICE_NAME="otel-contrib-collector"
PROCESS_NAME="otelcontribcol"

SERVICE_PATH="$FPM_DIR/$SERVICE_NAME.service"
PREINSTALL_PATH="$FPM_DIR/preinstall.sh"
POSTINSTALL_PATH="$FPM_DIR/postinstall.sh"
PREUNINSTALL_PATH="$FPM_DIR/preuninstall.sh"

docker_cp() {
local container="$1"
local src="$2"
local dest="$3"
local dest_dir="$( dirname "$dest" )"

echo "Copying $src to $container:$dest ..."
docker exec $container mkdir -p "$dest_dir"
docker cp "$src" $container:"$dest"
}

install_pkg() {
local container="$1"
local pkg_path="$2"
local pkg_base=$( basename "$pkg_path" )

echo "Installing $pkg_base ..."
docker_cp $container "$pkg_path" /tmp/$pkg_base
if [[ "${pkg_base##*.}" = "deb" ]]; then
docker exec $container dpkg -i /tmp/$pkg_base
else
docker exec $container rpm -ivh /tmp/$pkg_base
fi
}

uninstall_pkg() {
local container="$1"
local pkg_type="$2"
local pkg_name="${3:-"$PKG_NAME"}"

echo "Uninstalling $pkg_name ..."
if [[ "$pkg_type" = "deb" ]]; then
docker exec $container dpkg -r $pkg_name
else
docker exec $container rpm -e $pkg_name
fi
}
22 changes: 22 additions & 0 deletions internal/buildscripts/packaging/fpm/deb/Dockerfile.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# A debian9 image with systemd enabled. Must be run with:
# `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags
FROM debian:9

RUN apt-get update &&\
apt-get install -yq ca-certificates procps systemd

ENV container docker
RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \
"systemd-tmpfiles-setup.service" ] || rm -f $i; done); \
rm -f /lib/systemd/system/multi-user.target.wants/*;\
rm -f /lib/systemd/system/local-fs.target.wants/*; \
rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
rm -f /lib/systemd/system/anaconda.target.wants/*;

RUN systemctl set-default multi-user.target
ENV init /lib/systemd/systemd

VOLUME [ "/sys/fs/cgroup" ]

ENTRYPOINT ["/lib/systemd/systemd"]
14 changes: 14 additions & 0 deletions internal/buildscripts/packaging/fpm/deb/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Build otel-contrib-collector deb package

Build the otel-contrib-collector deb package with [fpm](https://github.com/jordansissel/fpm).

To build the deb package, run `make deb-package` from the repo root directory. The deb package will be written to
`dist/otel-contrib-collector_<version>_<arch>.deb`.

By default, `<arch>` is `amd64` and `<version>` is the latest git tag with `-post` appended, e.g. `1.2.3-post`.
To override these defaults, set the `ARCH` and `VERSION` environment variables, e.g.
`ARCH=arm64 VERSION=4.5.6 make deb-package`.

Run `./internal/buildscripts/packaging/fpm/test.sh PATH_TO_DEB_FILE [PATH_TO_CONFIG_FILE]` to run a basic installation
test with the built package. `PATH_TO_CONFIG_FILE` defaults to `examples/tracing/otel-collector-config.yml` if one is
not specified.
35 changes: 35 additions & 0 deletions internal/buildscripts/packaging/fpm/deb/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/bin/bash

set -euxo pipefail

SCRIPT_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )"
REPO_DIR="$( cd "$SCRIPT_DIR/../../../../../" && pwd )"
VERSION="${1:-}"
ARCH="${2:-"amd64"}"
OUTPUT_DIR="${3:-"$REPO_DIR/dist/"}"
OTELCONTRIBCOL_PATH="$REPO_DIR/bin/otelcontribcol_linux_$ARCH"

. $SCRIPT_DIR/../common.sh

if [[ -z "$VERSION" ]]; then
latest_tag="$( git describe --abbrev=0 --match v[0-9]* )"
VERSION="${latest_tag}-post"
fi

mkdir -p "$OUTPUT_DIR"

fpm -s dir -t deb -n $PKG_NAME -v ${VERSION#v} -f -p "$OUTPUT_DIR" \
--vendor "$PKG_VENDOR" \
--maintainer "$PKG_MAINTAINER" \
--description "$PKG_DESCRIPTION" \
--license "$PKG_LICENSE" \
--url "$PKG_URL" \
--architecture "$ARCH" \
--deb-dist "stable" \
--deb-user "$PKG_USER" \
--deb-group "$PKG_GROUP" \
--before-install "$PREINSTALL_PATH" \
--after-install "$POSTINSTALL_PATH" \
--pre-uninstall "$PREUNINSTALL_PATH" \
$SERVICE_PATH=/lib/systemd/system/$SERVICE_NAME.service \
$OTELCONTRIBCOL_PATH=/usr/bin/otelcontribcol
14 changes: 14 additions & 0 deletions internal/buildscripts/packaging/fpm/otel-contrib-collector.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
[Unit]
Description=OpenTelemety Contrib Collector
After=network.target

[Service]
ExecStart=/usr/bin/otelcontribcol --config /etc/otel-contrib-collector/config.yaml
KillMode=mixed
Restart=on-failure
Type=simple
User=otel
Group=otel

[Install]
WantedBy=multi-user.target
8 changes: 8 additions & 0 deletions internal/buildscripts/packaging/fpm/postinstall.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/sh

if command -v systemctl >/dev/null 2>&1; then
systemctl enable otel-contrib-collector.service
if [ -f /etc/otel-contrib-collector/config.yaml ]; then
systemctl start otel-contrib-collector.service
fi
fi
3 changes: 3 additions & 0 deletions internal/buildscripts/packaging/fpm/preinstall.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/sh

getent passwd otel >/dev/null || useradd --system --user-group --no-create-home --shell /sbin/nologin otel
Loading

0 comments on commit d1598f3

Please sign in to comment.