ci: switch build to GitHub Actions

.github/workflows/ci.yml

@@ -5,104 +5,195 @@ on:
  branches:
  - master
  schedule:
-  - cron: '00 01 * * *'
+ - cron: '00 01 * * *'
 jobs:
  test:
  name: test
- runs-on: ubuntu-18.04
+ env:
+ # For some builds, we use cross to test on 32-bit and big-endian
+ # systems.
+ CARGO: cargo
+ # When CARGO is set to CROSS, this is set to `--target matrix.target`.
+ TARGET_FLAGS:
+ # When CARGO is set to CROSS, TARGET_DIR includes matrix.target.
+ TARGET_DIR: ./target
+ # Emit backtraces on panics.
+ RUST_BACKTRACE: 1
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ build:
+ # We test ripgrep on a pinned version of Rust, along with the moving
+ # targets of 'stable' and 'beta' for good measure.
+ - pinned
+ - stable
+ - beta
+ # Our release builds are generated by a nightly compiler to take
+ # advantage of the latest optimizations/compile time improvements. So
+ # we test all of them here. (We don't do mips releases, but test on
+ # mips for big-endian coverage.)
+ - nightly
+ - nightly-musl
+ - nightly-32
+ - nightly-mips
+ - nightly-arm
+ - macos
+ - win-msvc
+ - win-gnu
+ include:
+ - build: pinned
+ os: ubuntu-18.04
+ rust: 1.41.0
+ - build: stable
+ os: ubuntu-18.04
+ rust: stable
+ - build: beta
+ os: ubuntu-18.04
+ rust: beta
+ - build: nightly
+ os: ubuntu-18.04
+ rust: nightly
+ - build: nightly-musl
+ os: ubuntu-18.04
+ rust: nightly
+ target: x86_64-unknown-linux-musl
+ - build: nightly-32
+ os: ubuntu-18.04
+ rust: nightly
+ target: i686-unknown-linux-gnu
+ - build: nightly-mips
+ os: ubuntu-18.04
+ rust: nightly
+ target: mips64-unknown-linux-gnuabi64
+ - build: nightly-arm
+ os: ubuntu-18.04
+ rust: nightly
+ # For stripping release binaries:
+ # docker run --rm -v $PWD/target:/target:Z \
+ # rustembedded/cross:arm-unknown-linux-gnueabihf \
+ # arm-linux-gnueabihf-strip \
+ # /target/arm-unknown-linux-gnueabihf/debug/rg
+ target: arm-unknown-linux-gnueabihf
+ - build: macos
+ os: macos-latest
+ rust: nightly
+ - build: win-msvc
+ os: windows-2019
+ rust: nightly
+ - build: win-gnu
+ os: windows-2019
+ rust: nightly-x86_64-gnu
  steps:
- - name: no-op
- run: echo hello
+ - name: Checkout repository
+ uses: actions/checkout@v1
+ with:
+ fetch-depth: 1
+
+ - name: Install packages (Ubuntu)
+ if: matrix.os == 'ubuntu-18.04'
+ run: |
+ ci/ubuntu-install-packages
+
+ - name: Install packages (macOS)
+ if: matrix.os == 'macos-latest'
+ run: |
+ ci/macos-install-packages
+
+ - name: Install Rust
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: ${{ matrix.rust }}
+ profile: minimal
+ override: true
+
+ - name: Use Cross
+ if: matrix.target != ''
+ run: |
+ # FIXME: to work around bugs in latest cross release, install master.
+ # See: https://github.com/rust-embedded/cross/issues/357
+ cargo install --git https://github.com/rust-embedded/cross
+ echo "::set-env name=CARGO::cross"
+ echo "::set-env name=TARGET_FLAGS::--target ${{ matrix.target }}"
+ echo "::set-env name=TARGET_DIR::./target/${{ matrix.target }}"
+
+ - name: Show command used for Cargo
+ run: |
+ echo "cargo command is: ${{ env.CARGO }}"
+ echo "target flag is: ${{ env.TARGET_FLAGS }}"
+
+ - name: Build ripgrep and all crates
+ run: ${{ env.CARGO }} build --verbose --all ${{ env.TARGET_FLAGS }}
+
+ - name: Build ripgrep with PCRE2
+ run: ${{ env.CARGO }} build --verbose --all --features pcre2 ${{ env.TARGET_FLAGS }}
 
-# test:
-# name: test
-# runs-on: ${{ matrix.os }}
-# strategy:
-# matrix:
-# # The docs seem to suggest that we can have a matrix with just an
-# # include directive, but it result in a "matrix must define at least
-# # one vector" error in the CI system.
-# build:
-# # - pinned-glibc
-# - pinned-musl
-# - stable
-# # - beta
-# # We test musl with nightly because every once in a while, this will
-# # catch an upstream regression.
-# # - nightly-glibc
-# # - nightly-musl
-# # - macos
-# # - win-msvc-32
-# # - win-msvc-64
-# # - win-gnu-32
-# # - win-gnu-64
-# include:
-# # - build: pinned-glibc
-# # os: ubuntu-18.04
-# # rust: 1.34.0
-# # target: x86_64-unknown-linux-gnu
-# - build: pinned-musl
-# os: ubuntu-18.04
-# rust: 1.34.0
-# target: x86_64-unknown-linux-musl
-# - build: stable
-# os: ubuntu-18.04
-# rust: stable
-# target: x86_64-unknown-linux-gnu
-# # - build: beta
-# # os: ubuntu-18.04
-# # rust: beta
-# # target: x86_64-unknown-linux-gnu
-# # - build: nightly-glibc
-# # os: ubuntu-18.04
-# # rust: nightly
-# # target: x86_64-unknown-linux-gnu
-# # - build: nightly-musl
-# # os: ubuntu-18.04
-# # rust: nightly
-# # target: x86_64-unknown-linux-musl
-# # - build: macos
-# # os: macOS-10.14
-# # rust: stable
-# # target: x86_64-apple-darwin
-# # - build: win-msvc-32
-# # os: windows-2019
-# # rust: stable
-# # target: i686-pc-windows-msvc
-# # - build: win-msvc-64
-# # os: windows-2019
-# # rust: stable
-# # target: x86_64-pc-windows-msvc
-# # - build: win-gnu-32
-# # os: windows-2019
-# # rust: stable-i686-gnu
-# # target: i686-pc-windows-gnu
-# # - build: win-gnu-64
-# # os: windows-2019
-# # rust: stable-x86_64-gnu
-# # target: x86_64-pc-windows-gnu
-# steps:
-# - name: Checkout repository
-# uses: actions/checkout@v1
-# with:
-# fetch-depth: 1
-# - name: Install Rust
-# uses: hecrj/setup-rust-action@v1
-# with:
-# rust-version: ${{ matrix.rust }}
-# - name: Install Rust Target
-# run: rustup target add ${{ matrix.target }}
-# - name: Install musl-gcc
-# if: contains(matrix.target, 'musl')
-# run: |
-# sudo apt-get install musl-tools
-# - name: Build everything
-# run: cargo build --verbose --target ${{ matrix.target }} --all --features pcre2
-# - name: Install zsh
-# if: matrix.build == 'stable'
-# run: sudo apt-get install zsh
-# - name: Test zsh auto-completions
-# if: matrix.build == 'stable'
-# run: ./ci/test_complete.sh
-# - name: Run tests
-# run: cargo test --verbose --target ${{ matrix.target }} --all --features pcre2
+ # This is useful for debugging problems when the expected build artifacts
+ # (like shell completions and man pages) aren't generated.
+ - name: Show build.rs stderr
+ shell: bash
+ run: |
+ set +x
+ stderr="$(find "${{ env.TARGET_DIR }}/debug" -name stderr -print0 | xargs -0 ls -t | head -n1)"
+ if [ -s "$stderr" ]; then
+ echo "===== $stderr ===== "
+ cat "$stderr"
+ echo "====="
+ fi
+ set -x
+
+ - name: Run tests with PCRE2 (sans cross)
+ if: matrix.target == ''
+ run: ${{ env.CARGO }} test --verbose --all --features pcre2 ${{ env.TARGET_FLAGS }}
+
+ - name: Run tests without PCRE2 (with cross)
+ # These tests should actually work, but they almost double the runtime.
+ # Every integration test spins up qemu to run 'rg', and when PCRE2 is
+ # enabled, every integration test is run twice: one with the default
+ # regex engine and once with PCRE2.
+ if: matrix.target != ''
+ run: ${{ env.CARGO }} test --verbose --all ${{ env.TARGET_FLAGS }}
+
+ - name: Test for existence of build artifacts (Windows)
+ if: matrix.os == 'windows-2019'
+ shell: bash
+ run: |
+ outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")"
+ ls "$outdir/_rg.ps1" && file "$outdir/_rg.ps1"
+
+ - name: Test for existence of build artifacts (Unix)
+ if: matrix.os != 'windows-2019'
+ shell: bash
+ run: |
+ outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")"
+ for f in rg.bash rg.fish rg.1; do
+ # We could use file -E here, but it isn't supported on macOS.
+ ls "$outdir/$f" && file "$outdir/$f"
+ done
+
+ - name: Test zsh shell completions (Unix, sans cross)
+ # We could test this when using Cross, but we'd have to execute the
+ # 'rg' binary (done in test_complete.sh) with qemu, which is a pain and
+ # doesn't really gain us much. If shell completion works in one place,
+ # it probably works everywhere.
+ if: matrix.target == '' && matrix.os != 'windows-2019'
+ shell: bash
+ run: ci/test_complete.sh
+
+ rustfmt:
+ name: rustfmt
+ runs-on: ubuntu-18.04
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v1
+ with:
+ fetch-depth: 1
+ - name: Install Rust
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ override: true
+ profile: minimal
+ components: rustfmt
+ - name: Check formatting
+ run: |
+ cargo fmt --all -- --check

Cross.toml

@@ -1,2 +1,11 @@
 [target.x86_64-unknown-linux-musl]
 image = "burntsushi/cross:x86_64-unknown-linux-musl"
+
+[target.i686-unknown-linux-gnu]
+image = "burntsushi/cross:i686-unknown-linux-gnu"
+
+[target.mips64-unknown-linux-gnuabi64]
+image = "burntsushi/cross:mips64-unknown-linux-gnuabi64"
+
+[target.arm-unknown-linux-gnueabihf]
+image = "burntsushi/cross:arm-unknown-linux-gnueabihf"

ci/cargo-out-dir

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Finds Cargo's `OUT_DIR` directory from the most recent build.
+#
+# This requires one parameter corresponding to the target directory
+# to search for the build output.
+
+if [ $# != 1 ]; then
+ echo "Usage: $(basename "$0") <target-dir>" >&2
+ exit 2
+fi
+
+# This works by finding the most recent stamp file, which is produced by
+# every ripgrep build.
+target_dir="$1"
+find "$target_dir" -name ripgrep-stamp -print0 \
+ | xargs -0 ls -t \
+ | head -n1 \
+ | xargs dirname

ci/docker/README.md

@@ -0,0 +1,24 @@
+These are Docker images used for cross compilation in CI builds (or locally)
+via the [Cross](https://github.com/rust-embedded/cross) tool.
+
+The Cross tool actually provides its own Docker images, and all Docker images
+in this directory are derived from one of them. We provide our own in order
+to customize the environment. For example, we need to install some things like
+`asciidoc` in order to generate man pages. We also install compression tools
+like `xz` so that tests for the `-z/--search-zip` flag are run.
+
+If you make a change to a Docker image, then you can re-build it. `cd` into the
+directory containing the `Dockerfile` and run:
+
+ $ cd x86_64-unknown-linux-musl
+ $ ./build
+
+At this point, subsequent uses of `cross` will now use your built image since
+Docker prefers local images over remote images. In order to make these changes
+stick, they need to be pushed to Docker Hub:
+
+ $ docker push burntsushi/cross:x86_64-unknown-linux-musl
+
+Of course, only I (BurntSushi) can push to that location. To make `cross` use
+a different location, then edit `Cross.toml` in the root of this repo to use
+a different image name for the desired target.

ci/docker/arm-unknown-linux-gnueabihf/Dockerfile

@@ -0,0 +1,4 @@
+FROM rustembedded/cross:arm-unknown-linux-gnueabihf
+
+COPY stage/ubuntu-install-packages /
+RUN /ubuntu-install-packages

ci/docker/arm-unknown-linux-gnueabihf/build

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+mkdir -p stage
+cp ../../ubuntu-install-packages ./stage/
+docker build -t burntsushi/cross:arm-unknown-linux-gnueabihf .

ci/docker/i686-unknown-linux-gnu/Dockerfile

@@ -0,0 +1,4 @@
+FROM rustembedded/cross:i686-unknown-linux-gnu
+
+COPY stage/ubuntu-install-packages /
+RUN /ubuntu-install-packages

ci/docker/i686-unknown-linux-gnu/build

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+mkdir -p stage
+cp ../../ubuntu-install-packages ./stage/
+docker build -t burntsushi/cross:i686-unknown-linux-gnu .

ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile

@@ -0,0 +1,4 @@
+FROM rustembedded/cross:mips64-unknown-linux-gnuabi64
+
+COPY stage/ubuntu-install-packages /
+RUN /ubuntu-install-packages

ci/docker/mips64-unknown-linux-gnuabi64/build

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+mkdir -p stage
+cp ../../ubuntu-install-packages ./stage/
+docker build -t burntsushi/cross:mips64-unknown-linux-gnuabi64 .

ci/docker/x86_64-unknown-linux-musl/Dockerfile

@@ -1,5 +1,4 @@
 FROM rustembedded/cross:x86_64-unknown-linux-musl
 
-RUN apt-get update \
- && apt-get install -y --no-install-recommends \
- libxslt1-dev asciidoc docbook-xsl xsltproc libxml2-utils
+COPY stage/ubuntu-install-packages /
+RUN /ubuntu-install-packages

ci/docker/x86_64-unknown-linux-musl/build

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+mkdir -p stage
+cp ../../ubuntu-install-packages ./stage/
+docker build -t burntsushi/cross:x86_64-unknown-linux-musl .

ci/macos-install-packages

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+brew install asciidoc docbook-xsl