Ghost queue

.cargo/config.toml

@@ -0,0 +1,2 @@
+[target.'cfg(all())']
+rustflags = ["--cfg", "tokio_unstable"]

.config/hakari.toml

@@ -0,0 +1,29 @@
+# This file contains settings for `cargo hakari`.
+# See https://docs.rs/cargo-hakari/latest/cargo_hakari/config for a full list of options.
+
+hakari-package = "foyer-workspace-hack"
+
+# Format version for hakari's output. Version 4 requires cargo-hakari 0.9.22 or above.
+dep-format-version = "4"
+
+# Setting workspace.resolver = "2" in the root Cargo.toml is HIGHLY recommended.
+# Hakari works much better with the new feature resolver.
+# For more about the new feature resolver, see:
+# https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver
+resolver = "2"
+
+# Add triples corresponding to platforms commonly used by developers here.
+# https://doc.rust-lang.org/rustc/platform-support.html
+platforms = [
+ # "x86_64-unknown-linux-gnu",
+ # "x86_64-apple-darwin",
+ # "x86_64-pc-windows-msvc",
+]
+
+# Write out exact versions rather than a semver range. (Defaults to false.)
+# exact-versions = true
+
+[traversal-excludes]
+third-party = [
+ { name = "miniz_oxide" },
+]

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+ - package-ecosystem: "cargo" # See documentation for possible values
+ directory: "/" # Location of package manifests
+ schedule:
+ interval: "daily"
+ rebase-strategy: "disabled"

.github/pull_request_template.md

@@ -0,0 +1,11 @@
+## What's changed and what's your intention?
+
+> Please explain **IN DETAIL** what the changes are in this PR and why they are needed. :D
+
+## Checklist
+
+- [ ] I have written the necessary rustdoc comments
+- [ ] I have added the necessary unit tests and integration tests
+- [ ] I have passed `make all` (or `make fast` instead if the old tests are not modified) in my local environment.
+
+## Related issues or PRs (optional)

.github/semantic.yml

@@ -0,0 +1,16 @@
+# Ref: https://github.com/zeke/semantic-pull-requests#configuration .
+titleAndCommits: true
+anyCommit: true
+types:
+ - feat
+ - fix
+ - docs
+ - style
+ - refactor
+ - perf
+ - test
+ - build
+ - ci
+ - chore
+ - revert
+allowMergeCommits: true

.github/template/generate.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# You will need to install yq >= 4.16 to use this tool.
+# brew install yq
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR"
+
+
+HEADER="""
+# ================= THIS FILE IS AUTOMATICALLY GENERATED =================
+#
+# Please run generate.sh and commit after editing the workflow templates.
+#
+# ========================================================================
+"""
+
+# Generate workflow for main branch
+echo "$HEADER" > ../workflows/main.yml
+# shellcheck disable=SC2016
+yq ea '. as $item ireduce ({}; . * $item )' template.yml main-override.yml | yq eval '... comments=""' - >> ../workflows/main.yml
+echo "$HEADER" >> ../workflows/main.yml
+
+# Generate workflow for pull requests
+echo "$HEADER" > ../workflows/pull-request.yml
+# shellcheck disable=SC2016
+yq ea '. as $item ireduce ({}; . * $item )' template.yml pr-override.yml | yq eval '... comments=""' - >> ../workflows/pull-request.yml
+echo "$HEADER" >> ../workflows/pull-request.yml
+
+if [ "$1" == "--check" ] ; then
+ if ! git diff --exit-code; then
+ echo "Please run generate.sh and commit after editing the workflow templates."
+ exit 1
+ fi
+fi

.github/template/main-override.yml

@@ -0,0 +1,6 @@
+name: CI (main)
+
+on:
+ push:
+ branches: [main]
+ workflow_dispatch:

.github/template/pr-override.yml

@@ -0,0 +1,9 @@
+name: CI
+
+on:
+ pull_request:
+ branches: [main]
+
+concurrency:
+ group: environment-${{ github.ref }}
+ cancel-in-progress: true

.github/template/template.yml

@@ -0,0 +1,236 @@
+name:
+
+on:
+
+env:
+ RUST_TOOLCHAIN: nightly-2023-12-26
+ CARGO_TERM_COLOR: always
+ CACHE_KEY_SUFFIX: 20240306
+
+jobs:
+ misc-check:
+ name: misc check
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Run typos check
+ uses: crate-ci/typos@master
+ - name: Install yq
+ run: |
+ wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
+ env:
+ YQ_VERSION: v4.16.1
+ BINARY: yq_linux_amd64
+ BUF_VERSION: 1.0.0-rc6
+ - name: Install jq
+ uses: dcarbone/[email protected]
+ - name: Check if CI workflows are up-to-date
+ run: |
+ ./.github/template/generate.sh --check
+ - name: Check if Grafana dashboards are minimized
+ run: |
+ ./scripts/minimize-dashboards.sh --check
+ - name: Run ShellCheck
+ uses: ludeeus/action-shellcheck@master
+ rust-test:
+ name: rust test with codecov
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Install rust toolchain@v1
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: ${{ env.RUST_TOOLCHAIN }}
+ components: rustfmt, clippy, llvm-tools-preview
+ - name: Cache Cargo home
+ uses: actions/cache@v2
+ id: cache
+ with:
+ path: |
+ ~/.cargo/bin/
+ ~/.cargo/registry/index/
+ ~/.cargo/registry/cache/
+ ~/.cargo/git/db/
+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-rust-test
+ - name: Install cargo-binstall
+ if: steps.cache.outputs.cache-hit != 'true'
+ run: |
+ curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
+ - name: Install cargo tools
+ if: steps.cache.outputs.cache-hit != 'true'
+ run: |
+ cargo binstall -y cargo-sort cargo-hakari
+ - name: Run rust cargo-sort check
+ run: |
+ cargo sort -w -c
+ - name: Run hakari check
+ run: |
+ cargo hakari generate --diff
+ cargo hakari manage-deps --dry-run
+ - name: Run rust format check
+ run: |
+ cargo fmt --all -- --check
+ - name: Run rust clippy check
+ run: |
+ cargo clippy --all-targets --features tokio-console -- -D warnings
+ cargo clippy --all-targets --features deadlock -- -D warnings
+ cargo clippy --all-targets -- -D warnings
+ - if: steps.cache.outputs.cache-hit != 'true'
+ uses: taiki-e/install-action@cargo-llvm-cov
+ - if: steps.cache.outputs.cache-hit != 'true'
+ uses: taiki-e/install-action@nextest
+ - name: Run rust test with coverage (igored tests)
+ run: |
+ cargo llvm-cov --no-report nextest --run-ignored ignored-only --no-capture --workspace
+ - name: Run rust test with coverage
+ run: |
+ cargo llvm-cov --no-report nextest
+ - name: Generate codecov report
+ run: |
+ cargo llvm-cov report --lcov --output-path lcov.info
+ - uses: codecov/codecov-action@v3
+ with:
+ verbose: true
+ deadlock:
+ name: run with single worker thread and deadlock detection
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Install rust toolchain@v1
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: ${{ env.RUST_TOOLCHAIN }}
+ - name: Cache Cargo home
+ uses: actions/cache@v2
+ id: cache
+ with:
+ path: |
+ ~/.cargo/bin/
+ ~/.cargo/registry/index/
+ ~/.cargo/registry/cache/
+ ~/.cargo/git/db/
+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
+ - name: Run foyer-storage-bench with single worker thread and deadlock detection
+ env:
+ RUST_BACKTRACE: 1
+ RUSTFLAGS: "--cfg tokio_unstable"
+ RUST_LOG: info
+ TOKIO_WORKER_THREADS: 1
+ run: |-
+ cargo build --all --features deadlock
+ mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
+ timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+ asan:
+ name: run with address saniziter
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Install rust toolchain@v1
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: ${{ env.RUST_TOOLCHAIN }}
+ - name: Cache Cargo home
+ uses: actions/cache@v2
+ id: cache
+ with:
+ path: |
+ ~/.cargo/bin/
+ ~/.cargo/registry/index/
+ ~/.cargo/registry/cache/
+ ~/.cargo/git/db/
+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
+ - name: Run Unit Tests With Address Sanitizer
+ env:
+ RUST_BACKTRACE: 1
+ RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
+ RUST_LOG: info
+ run: |-
+ cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+ - name: Run foyer-storage-bench With Address Sanitizer
+ env:
+ RUST_BACKTRACE: 1
+ RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
+ RUST_LOG: info
+ run: |-
+ cargo build --all --target x86_64-unknown-linux-gnu
+ mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
+ timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+ lsan:
+ name: run with leak saniziter
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Install rust toolchain@v1
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: ${{ env.RUST_TOOLCHAIN }}
+ - name: Cache Cargo home
+ uses: actions/cache@v2
+ id: cache
+ with:
+ path: |
+ ~/.cargo/bin/
+ ~/.cargo/registry/index/
+ ~/.cargo/registry/cache/
+ ~/.cargo/git/db/
+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-lsan
+ - name: Run Unit Tests With Leak Sanitizer
+ env:
+ RUST_BACKTRACE: 1
+ RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+ RUST_LOG: info
+ run: |-
+ cargo test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+ - name: Run foyer-storage-bench With Leak Sanitizer
+ env:
+ RUST_BACKTRACE: 1
+ RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+ RUST_LOG: info
+ run: |-
+ cargo build --all --target x86_64-unknown-linux-gnu
+ mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
+ timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+ deterministic-test:
+ name: run deterministic test
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Install rust toolchain@v1
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: ${{ env.RUST_TOOLCHAIN }}
+ components: rustfmt, clippy
+ - name: Cache Cargo home
+ uses: actions/cache@v2
+ id: cache
+ with:
+ path: |
+ ~/.cargo/bin/
+ ~/.cargo/registry/index/
+ ~/.cargo/registry/cache/
+ ~/.cargo/git/db/
+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
+ - if: steps.cache.outputs.cache-hit != 'true'
+ uses: taiki-e/install-action@nextest
+ - name: Run rust clippy check (madsim)
+ env:
+ RUST_BACKTRACE: 1
+ RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+ RUST_LOG: info
+ TOKIO_WORKER_THREADS: 1
+ run: |-
+ cargo clippy --all-targets
+ - name: Run nexmark test (madsim)
+ env:
+ RUST_BACKTRACE: 1
+ RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+ RUST_LOG: info
+ TOKIO_WORKER_THREADS: 1
+ run: |-
+ cargo nextest run --all

.github/workflows/license_check.yml

@@ -0,0 +1,19 @@
+name: License Checker
+
+on:
+ push:
+ branches:
+ - main
+ - "forks/*"
+ pull_request:
+ branches: 
+ - main
+ - "v*.*.*-rc"
+jobs:
+ license-header-check:
+ runs-on: ubuntu-latest
+ name: license-header-check
+ steps:
+ - uses: actions/checkout@v3
+ - name: Check License Header
+ uses: apache/skywalking-eyes/header@df70871af1a8109c9a5b1dc824faaf65246c5236