repro.in

#!/usr/bin/bash

VERSION='REPRO_VERSION'

set -eE -o pipefail

if ((DEBUG)); then
  set -x
fi

# Ensure umask is set correctly
umask 022

BUILDDIRECTORY=/var/lib/repro

KEYRINGCACHE="${BUILDDIRECTORY}/keyring"

BOOTSTRAPMIRROR="https://geo.mirror.pkgbuild.com/iso/latest"
BOOTSTRAP_IMG=archlinux-bootstrap-"$(uname -m)".tar
CONFIGDIR='REPRO_CONFIG_DIR'

HOSTMIRROR="https://geo.mirror.pkgbuild.com/\$repo/os/\$arch"

ARCHIVEURL="${ARCH_ARCHIVE_CACHE:-https://archive.archlinux.org/packages}"

IMGDIRECTORY=$(mktemp -dt XXXXXXXXXX.arch_img)
trap "{ rm -r $IMGDIRECTORY; }" EXIT

DIFFOSCOPE="diffoscope"

# Turn on/off check in repro
NOCHECK=${NOCHECK:-0}

# Check if systemd >=242
if [ $(systemd-nspawn --version | grep -m 1 -Eo '[0-9]+' | head -1) -ge 242 ]; then
    ISSYSTEMD242=1
fi

CACHEDIR="${CACHEDIR:-cache}"
OUTDIR="${OUTDIR:-./build}"

# Default options
run_diffoscope=0

# By default we don't assume a PKGBUILD
pkgbuild_file=0

makepkg_args=(
    --syncdeps
    --clean
    --noconfirm
    --skippgpcheck
)

# Desc: Escalates privileges
orig_argv=("$0" "$@")
src_owner=${SUDO_USER:-$USER}
function check_root() {
    local keepenv=$1
    (( EUID == 0 )) && return
    if type -P sudo >/dev/null; then
        exec sudo --preserve-env=$keepenv -- "${orig_argv[@]}"
    else
        exec su root -c "$(printf ' %q' "${orig_argv[@]}")"
    fi
}

# Use a private gpg keyring
function gpg() {
  command gpg --homedir="$BUILDDIRECTORY/_gnupg" "$@"
}

function init_gnupg() {
    mkdir -p "$BUILDDIRECTORY/"
    mkdir -p --mode 700 "$BUILDDIRECTORY/_gnupg"

    # ensure signing key is available
    # We try WKD first, then fallback to keyservers.
    # This works on debian./
    gpg --keyserver=p80.pool.sks-keyservers.net --auto-key-locate wkd,keyserver --locate-keys pierre@archlinux.org
}

# Desc: Sets the appropriate colors for output
function colorize() {
    # test if stdout is a tty
    if [ -t 1 ]; then
        # prefer terminal safe colored and bold text when tput is supported
        if tput setaf 0 &>/dev/null; then
            ALL_OFF="$(tput sgr0)"
            BOLD="$(tput bold)"
            BLUE="${BOLD}$(tput setaf 4)"
            GREEN="${BOLD}$(tput setaf 2)"
            RED="${BOLD}$(tput setaf 1)"
            YELLOW="${BOLD}$(tput setaf 3)"
        else
            ALL_OFF="\e[0m"
            BOLD="\e[1m"
            BLUE="${BOLD}\e[34m"
            GREEN="${BOLD}\e[32m"
            RED="${BOLD}\e[31m"
            YELLOW="${BOLD}\e[33m"
        fi
    else
        # stdout is piped, disable colors
        ALL_OFF=""
        BOLD=""
        BLUE=""
        GREEN=""
        RED=""
        YELLOW=""
    fi
    readonly ALL_OFF BOLD BLUE GREEN RED YELLOW
}

# Desc: Message format
function msg() {
    local mesg=$1; shift
    # shellcheck disable=SC2059
    printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2
}

# Desc: Sub-message format
function msg2() {
    local mesg=$1; shift
    # shellcheck disable=SC2059
    printf "${BLUE}  ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2
}

# Desc: Warning format
function warning() {
    local mesg=$1; shift
    # shellcheck disable=SC2059
    printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2
}

# Desc: Error format
function error() {
    local mesg=$1; shift
    # shellcheck disable=SC2059
    printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2
}

##
#  usage : nlock( $fd, $file, $message, [ $message_arguments... ] )
#
# Desc: non-blocking exclusive (write) lock
##
nlock() {
    # Only reopen the FD if it wasn't handed to us
    if ! [[ "/dev/fd/$1" -ef "$2" ]]; then
        mkdir -p -- "$(dirname -- "$2")"
        eval "exec $1>"'"$2"'
    fi

    flock -n "$1"
}

##
#  usage : lock( $fd, $file, $message, [ $message_arguments... ] )
#
# Desc: normal - blocking exclusive (write) lock
##
lock() {
    # Only reopen the FD if it wasn't handed to us
    if ! [[ "/dev/fd/$1" -ef "$2" ]]; then
        mkdir -p -- "$(dirname -- "$2")"
        eval "exec $1>"'"$2"'
    fi

    flock "$1"
}

##
#  usage : slock( $fd, $file, $message, [ $message_arguments... ] )
#
# Desc: blocking shared (read) lock
##
slock() {
    # Only reopen the FD if it wasn't handed to us
    if ! [[ "/dev/fd/$1" -ef "$2" ]]; then
        mkdir -p -- "$(dirname -- "$2")"
        eval "exec $1>"'"$2"'
    fi

    flock -s "$1"
}

##
#  usage : lock_close( $fd )
##
lock_close() {
    local fd=$1
    # https://github.com/koalaman/shellcheck/issues/862
    # shellcheck disable=2034
    exec {fd}>&-
}

# Desc: Executes an command inside a given nspawn container
# 1: Container name
# 2: Command to execute
function exec_nspawn(){
    local container=$1

    # EPHEMERAL in systemd-nspawn uses implicit overlayfs mounts to provide
    # the container. If the root container is being updated or files are in
    # the lower directory disappear the results are unspecified and might
    # cause weird behaviour.
    #
    # Thus we acquire read locks on the build container to ensure nothing gets
    # a write lock. The code is weird because the locking mechanism here is
    # implicit as opposed to explicit in the top level of cmd_check.
    if ((EPHEMERAL)); then
        slock 8 "$BUILDDIRECTORY/$container.lock"
    fi
    systemd-nspawn -q \
        --as-pid2 \
        --register=no \
        ${EPHEMERAL:+--ephemeral} \
        ${ISSYSTEMD242:+--pipe} \
        -E "PATH=/usr/local/sbin:/usr/local/bin:/usr/bin" \
        -D "$BUILDDIRECTORY/$container" "${@:2}"
    if ((EPHEMERAL)); then
        lock_close 8 "$BUILDDIRECTORY/$container.lock"
    fi
}

# Desc: Removes the root container
function cleanup_root_volume(){
    warning "Removing root container..."
    rm -rf "$BUILDDIRECTORY/root"
}

# Desc: Removes a given snapshot
# 1: Snapshot name
function remove_snapshot (){
    local build=$1
    msg2 "Delete snapshot for $build..."
    rm -rf "${BUILDDIRECTORY:?}/${build}"
    trap - ERR INT
}

# Desc: Creates a snapshot of the root container
# 1: name of container
function create_snapshot (){
    local build="$1"
    trap "{ remove_snapshot \"$build\" ; exit 1; }" ERR INT
    msg2 "Create snapshot for $build..."
    mkdir -p "${BUILDDIRECTORY}/${build}/startdir"
    touch "$BUILDDIRECTORY/$build"
}

# Desc: Build a package inside a container
# 1: Container name
# 2: Container buildpath
function build_package(){
    local build=$1
    local builddir=${2:-"/startdir"}
    local args=""
    if ((pkgbuild_file)); then
      args=--bind="${PWD}:/startdir"
    fi
    exec_nspawn "$build" \
bash <<-__END__
set -e
install -d -o builduser -g builduser /pkgdest
install -d -o builduser -g builduser /srcpkgdest
install -d -o builduser -g builduser /build
__END__
    exec_nspawn "$build" $args sudo -iu builduser bash -c ". /etc/profile; . /env; cd /startdir; makepkg ${makepkg_args[*]}"
    mkdir -p "$OUTDIR"
    for pkgfile in "$BUILDDIRECTORY/$build"/pkgdest/*; do
        mv "$pkgfile" "$OUTDIR/"
    done
    chown -R "$src_owner" "$OUTDIR"
}

# Desc: Sets up a container with the correct files
function init_chroot(){
    mkdir -p "$BUILDDIRECTORY"

    # Always lock first. Otherwise we might end up...
    # - doing the same thing again - if using test/lock/mkdir
    # - with empty directory in the follow-up lock - if using test/mkdir/lock
    lock 9 "$BUILDDIRECTORY"/root.lock
    if [ ! -d "$BUILDDIRECTORY"/root ]; then
        init_gnupg

        if ! compgen -G "$IMGDIRECTORY/$bootstrap_img"* > /dev/null; then
            msg "Downloading bootstrap image..."

            for ext in zst gz; do
                bootstrap_img="$BOOTSTRAP_IMG.$ext"
                ( cd "$IMGDIRECTORY" && curl -f --remote-name-all "$BOOTSTRAPMIRROR/$bootstrap_img"{,.sig} )
                if ! gpg --verify "$IMGDIRECTORY/$bootstrap_img.sig" "$IMGDIRECTORY/$bootstrap_img"; then
                    error "Can't verify image"
                    exit 1
                fi
                break
            done 
        fi

        msg "Preparing chroot"
        trap '{ cleanup_root_volume; exit 1; }' ERR
        trap '{ cleanup_root_volume; trap - INT; kill -INT $$; }' INT

        msg2 "Extracting image into container..."
        mkdir -p "$BUILDDIRECTORY/root"
        tar xvf "$IMGDIRECTORY/$bootstrap_img" -C "$BUILDDIRECTORY/root" --strip-components=1 > /dev/null

        printf 'Server = %s\n' "$HOSTMIRROR" > "$BUILDDIRECTORY"/root/etc/pacman.d/mirrorlist
        sed -i "s/LocalFileSigLevel.*//g" "$BUILDDIRECTORY/root/etc/pacman.conf"

        systemd-machine-id-setup --root="$BUILDDIRECTORY"/root
        msg2 "Setting up keyring, this might take a while..."
        exec_nspawn root pacman-key --init &> /dev/null
        exec_nspawn root pacman-key --populate archlinux &> /dev/null
        touch "$BUILDDIRECTORY/root/.repro-2"
    else
      if [ ! -f "$BUILDDIRECTORY/root/.repro-2" ]; then
        error "Please delete $BUILDDIRECTORY and initialize the chroots again"
        exit 1
      fi
      msg "Reusing existing container"
    fi
    lock_close 9

    if nlock 9 "$BUILDDIRECTORY"/root.lock; then
      msg "Updating container"
      printf 'Server = %s\n' "$HOSTMIRROR" > "$BUILDDIRECTORY"/root/etc/pacman.d/mirrorlist
      exec_nspawn root pacman -Sy --noconfirm archlinux-keyring
      exec_nspawn root pacman -Su --noconfirm
      lock_close 9
    else
      msg "Couldn't acquire container lock, didn't update."
    fi
    trap - ERR INT
}

# Desc: Reproduces a package
function cmd_check(){
    local cachedir="${CACHEDIR}"

    trap - ERR INT

    declare -A buildinfo
    while IFS=$'=' read -r key value; do
        [[ "${key}" = [#!]* ]] || [[ "${key}" = "" ]] || buildinfo["${key}"]="${value}"
    done <<< "$(buildinfo -ff "${pkg}")"
    packager="${buildinfo[packager]}"
    builddir="${buildinfo[builddir]}"
    pkgver="${buildinfo[pkgver]}"
    pkgbase=${buildinfo[pkgbase]}
    options=${buildinfo[options]}
    buildenv=${buildinfo[buildenv]}
    format=${buildinfo[format]}
    installed=${buildinfo[installed]}

    pkgbuild_sha256sum="${buildinfo[pkgbuild_sha256sum]}"
    SOURCE_DATE_EPOCH="${buildinfo[builddate]}"
    BUILDTOOL=${buildinfo[buildtool]}
    BUILDTOOLVER=${buildinfo[buildtoolver]}

    DEVTOOLS="current devtools (fallback)"
    DEVTOOLS_PKG="devtools"
    if [[ -z "${BUILDTOOL}" ]] || [[ "${BUILDTOOL}" = makepkg ]]; then
      DEVTOOLS="devtools-20210202-3-any"
      DEVTOOLS_PKG="$ARCHIVEURL/d/devtools/${DEVTOOLS}.pkg.tar.zst"
    elif [[ "${BUILDTOOL}" = devtools ]] ; then
      DEVTOOLS="${BUILDTOOL}-${BUILDTOOLVER}"
      DEVTOOLS_PKG="$ARCHIVEURL/${BUILDTOOL:0:1}/${DEVTOOLS}.pkg.tar${pkg##*tar}"
    fi
    msg2 "Using devtools version: %s" "${DEVTOOLS}"

    if [[ ${format} -ne 1 && ${format} -ne 2 ]]; then
      error "unsupported BUILDINFO format or no format definition found, aborting rebuild"
      exit 1
    fi

    msg2 "Preparing packages"
    mkdir -p "${cachedir}"
    mapfile -t packages < <(buildinfo -d "${cachedir}" "${pkg}")
    msg2 "Finished preparing packages"

    msg "Starting build..."
    local build="${pkgbase}_$$"
    create_snapshot "$build" 0

    local build_root_dir="$BUILDDIRECTORY/${build}"

    # Father I have sinned
    if ((!pkgbuild_file)); then
    msg2 "Fetching PKGBUILD from git..."

    # Lock the cachedir as we might have a race condition with pacman -S and the cachedir
    lock 9 "${cachedir}.lock"

    EPHEMERAL=1 exec_nspawn root --bind="${build_root_dir}/startdir:/startdir" --bind="$(readlink -e ${cachedir}):/var/cache/pacman/pkg" \
    bash <<-__END__
shopt -s globstar

pacman -S devtools --noconfirm --needed

if ! pkgctl repo clone --protocol https --switch "$pkgver" "$pkgbase"; then
    echo "ERROR: Failed checkout $pkgbase" >&2
    exit 1
fi

if ! echo "$pkgbuild_sha256sum  $pkgbase/PKGBUILD" | sha256sum -c; then
    echo "ERROR: Failed to find commit this was built with (PKGBUILD checksum didn't match)" >&2
    exit 1
fi

mv ./$pkgbase/* /startdir
__END__
    lock_close 9 "${cachedir}.lock"
  elif [[ -r "PKGBUILD" ]]; then
    if [[ "$(sha256sum PKGBUILD | awk '{print $1}')" != "$pkgbuild_sha256sum" ]]; then
      error "PKGBUILD doesn't match the checksum"
      exit 1
    fi
  else
    error "No PKGBUILD file present!"
    exit 1
  fi

    # buildinfo returns packages with absolute paths to the location
    # this strips the paths and adds "cache/" prefix
    packages=(${packages[@]##*/})
    packages=(${packages[@]/#/cache\/})

    # shellcheck disable=SC2086
    keyring_package="$(printf -- '%s\n' ${installed[*]} | grep -E "archlinux-keyring")"

    mkdir -p "$KEYRINGCACHE"

    # Always lock first. Otherwise we might end up...
    # - doing the same thing again - if using test/lock/mkdir
    # - with empty directory in the follow-up lock - if using test/mkdir/lock
    lock 9 "$KEYRINGCACHE/$keyring_package.lock"
    if [ ! -d "$KEYRINGCACHE/$keyring_package" ]; then
      msg2 "Setting up $keyring_package in keyring cache, this might take a while..."

      # shellcheck disable=SC2086
      keyring=$(printf -- '%s\n' ${packages[*]} | grep -E "archlinux-keyring")
      EPHEMERAL=1 exec_nspawn root --bind="${build_root_dir}:/mnt" --bind="$(readlink -e "${cachedir}"):/cache" bash -c \
          'pacstrap -U /mnt -dd "$@"' -bash "${keyring}" &>/dev/null

      mkdir -p "$KEYRINGCACHE/$keyring_package"
      trap "{ rm -rf $KEYRINGCACHE/$keyring_package ; exit 1; }" ERR INT

      # We have to rewind time for gpg when building a package so that
      # signatures which were valid at the time the package was built
      # are still considered valid now, even if e.g. one of the keys
      # has since expired.
      #
      # However, gpg is finicky about time. Signatures which appear to
      # be created in the future, or by a key created in the future,
      # will be ignored. Keys which appear to be created in the future
      # cannot be signed. To make things work we need to create the
      # local master key and sign everything at a time after every key
      # in the keyring exists, but before any packages that depend on
      # it could have been built.
      #
      # Do this by using precisely the time that the keyring package
      # was built.

      keyring_build_date="$(buildinfo -f builddate "${cachedir}/${keyring##*/}")"

      # Note that while we leave faked-system-time in gpg.conf, this
      # will be overridden during the actual build by adding another
      # faked-system-time line to the end of the file, which takes
      # precedence.
      EPHEMERAL=1 exec_nspawn root \
        --bind="$KEYRINGCACHE/$keyring_package:/mnt" \
        --bind="${build_root_dir}/usr/share/pacman/keyrings:/usr/share/pacman/keyrings" \
        -E PACMAN_KEYRING_DIR=/mnt \
        bash &> /dev/null <<-__END__
echo "faked-system-time ${keyring_build_date}" >> /mnt/gpg.conf
pacman-key --init
pacman-key --populate archlinux
__END__
      trap - ERR INT
    else
      msg2 "Found $keyring_package in keyring cache"
    fi
    lock_close 9 "$KEYRINGCACHE/$keyring_package.lock"

    # Acquire shared locks for keyring as it could still be initialized at this point
    slock 9 "$KEYRINGCACHE/$keyring_package.lock"
    msg "Installing packages"
    # shellcheck disable=SC2086
    EPHEMERAL=1 exec_nspawn root \
      --bind="${build_root_dir}:/mnt" \
      --bind-ro="$KEYRINGCACHE/$keyring_package:/gnupg" \
      --bind="$(readlink -e ${cachedir}):/var/cache/pacman/pkg" \
      --bind="$(readlink -e ${cachedir}):/cache" \
      bash -bash "${packages[@]}" <<-__END__
set -e
rm --recursive /etc/pacman.d/gnupg/
cp --target-directory=/etc/pacman.d/ --recursive /gnupg
echo "faked-system-time ${SOURCE_DATE_EPOCH}" >> /etc/pacman.d/gnupg/gpg.conf
pacstrap -G -U /mnt --needed "\$@"

echo "Installing devtools from $DEVTOOLS_PKG"
# Ignore all dependencies since we only want the file
# Saves us a few seconds and doesn't download a bunch of things
# we are getting rid off
if [[ "$DEVTOOLS_PKG" == https* ]]; then
  pacman --noconfirm --needed -Udd "$DEVTOOLS_PKG"
else
  pacman --noconfirm --needed -Sddu "$DEVTOOLS_PKG"
fi

if [[ -f /usr/share/devtools/makepkg.conf.d/x86_64.conf ]]; then
    cp -v /usr/share/devtools/makepkg.conf.d/x86_64.conf /mnt/etc/makepkg.conf
elif [[ -f /usr/share/devtools/makepkg-x86_64.conf ]]; then
    cp -v /usr/share/devtools/makepkg-x86_64.conf /mnt/etc/makepkg.conf
else
    echo "Failed to find the makepkg.conf location, please report to archlinux-repro"
    exit 1
fi
__END__
    lock_close 9 "$KEYRINGCACHE/$keyring_package.lock"

    # Setup makepkg.conf
    {
        printf 'MAKEFLAGS="%s"\n' "${MAKEFLAGS:--j$(nproc)}"
        printf 'PKGDEST=/pkgdest\n'
        printf 'SRCPKGDEST=/srcpkgdest\n'
        printf 'BUILDDIR=%s\n' "${builddir}"
        printf 'PACKAGER=%s\n' "${packager@Q}"
        printf 'OPTIONS=(%s)\n' "${options}"
        printf 'BUILDENV=(%s)\n' "${buildenv}"
        printf 'COMPRESSZST=(zstd -c -T0 --ultra -20 -)\n'
        printf 'PKGEXT=".pkg.tar%s"\n' "${pkg##*tar}"
     } >> "$build_root_dir/etc/makepkg.conf"

    # Setup environment variables for makepkg
    {
        printf 'export SOURCE_DATE_EPOCH="%s"\n' "${SOURCE_DATE_EPOCH}"
        printf 'export BUILDTOOL="%s"\n' "${BUILDTOOL}"
        printf 'export BUILDTOOLVER="%s"\n' "${BUILDTOOLVER}"
     } >> "$build_root_dir/env"

    printf '%s.UTF-8 UTF-8\n' en_US de_DE > "$build_root_dir/etc/locale.gen"
    printf 'LANG=en_US.UTF-8\n' > "$build_root_dir/etc/locale.conf"
    exec_nspawn "$build" locale-gen

    printf 'builduser ALL = NOPASSWD: /usr/bin/pacman\n' > "$build_root_dir/etc/sudoers.d/builduser-pacman"
    exec_nspawn "$build" useradd -m -s /bin/bash -d /build builduser
    echo "keyserver-options auto-key-retrieve" | install -Dm644 /dev/stdin "$build_root_dir/build/.gnupg/gpg.conf"
    exec_nspawn "$build" chown -R builduser /build/.gnupg /startdir
    exec_nspawn "$build" chmod 700 /build/.gnupg

    build_package "$build" "$builddir"
    remove_snapshot "$build"
    chown -R "$src_owner" "${cachedir}"

    msg "Comparing hashes..."
    if diff -q -- "$pkg" "$OUTDIR/$(basename "$pkg")" > /dev/null ; then
      msg "Package is reproducible!"
      exit 0
    else
      error "Package is not reproducible"
      if ((run_diffoscope)); then
          PYTHONIOENCODING=utf-8 $DIFFOSCOPE "$pkg" "$OUTDIR/$(basename "$pkg")" || true
      fi
      exit 1
    fi
}

# Desc: Prints the help section
function print_help() {
cat <<__END__
Usage:
  repro [options] <local-package>

General Options:
 -h                           Print this help message
 -d                           Run diffoscope if packages are not reproducible
 -f                           Use the local PKGBUILD for building
 -n                           Run makepkg with --nocheck
 -V                           Print version information
 -o <path>                    Set the output directory (default: ./build)
__END__
}

function print_version() {
    echo "repro ${VERSION}"
}

function parse_args() {
    while getopts :hdnfVo: arg; do
        case $arg in
            h) print_help; exit 0;;
            V) print_version; exit 0;;
            f) pkgbuild_file=1;;
            d) run_diffoscope=1;;
            n) NOCHECK=1;;
            o) OUTDIR="$OPTARG";;
            *) error "unknown argument ${OPTARG}" ; print_help; exit 1;;
        esac
    done

    if ((NOCHECK)); then
        makepkg_args+=(--nocheck)
    fi

    # Save command args (such as path to .pkg.tar.xz file)
    shift $((OPTIND-1))

    if [[ $# != 1 ]]; then
        error "too many packages provided"
        print_help
        exit 1
    fi

    pkg="$@"
    if [[ ! -f "${pkg}" ]]; then
        error "argument provided ${pkg} isn't a valid file"
        print_help
        exit 1
    fi
}

function source_conf() {
    local repro_conf
    local xdg_repro_dir

    repro_conf=$CONFIGDIR/repro.conf
    if [[ -r $repro_conf ]]; then
        # shellcheck source=/dev/null
        source "$repro_conf"
    fi

    xdg_repro_dir="${XDG_CONFIG_HOME:-$HOME/.config}/archlinux-repro"
    if [[ -r "$xdg_repro_dir/repro.conf" ]]; then
        # shellcheck source=/dev/null
        source "$xdg_repro_dir/repro.conf"
    elif [[ -r "$HOME/.repro.conf" ]]; then
        # shellcheck source=/dev/null
        source "$HOME/.repro.conf"
    fi
}

colorize
source_conf
parse_args "$@"
check_root NOCHECK,MAKEFLAGS,DEBUG,CACHEDIR
print_version
init_chroot
cmd_check