From 1762ae5eea09506a5f3f0fc18f2b0a681276628c Mon Sep 17 00:00:00 2001 From: Xenira <1288524+Xenira@users.noreply.github.com> Date: Thu, 11 Dec 2025 19:38:51 +0100 Subject: [PATCH 1/4] test(benches): add basic benchmark infrastructure Refs: #599 --- .cargo/config.toml | 3 ++ Cargo.toml | 2 +- benches/Cargo.toml | 24 +++++++++ benches/benches/binary_bench.rs | 84 +++++++++++++++++++++++++++++++ benches/benches/function_call.php | 11 ++++ benches/src/lib.rs | 43 ++++++++++++++++ flake.lock | 12 ++--- flake.nix | 2 + 8 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 benches/Cargo.toml create mode 100644 benches/benches/binary_bench.rs create mode 100644 benches/benches/function_call.php create mode 100644 benches/src/lib.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 288e4ef9b..88c4308c0 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -9,3 +9,6 @@ linker = "rust-lld" [target.'cfg(target_env = "musl")'] rustflags = ["-C", "target-feature=-crt-static"] + +[profile.bench] +debug = true diff --git a/Cargo.toml b/Cargo.toml index b07f582d0..94d2e34b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ runtime = ["bindgen/runtime"] static = ["bindgen/static"] [workspace] -members = ["crates/macros", "crates/cli", "tests"] +members = ["crates/macros", "crates/cli", "tests", "benches"] [package.metadata.docs.rs] rustdoc-args = ["--cfg", "docs"] diff --git a/benches/Cargo.toml b/benches/Cargo.toml new file mode 100644 index 000000000..22be4d28e --- /dev/null +++ b/benches/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "benches" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +ext-php-rs = { path = "../" } +gungraun = { version = "0.17.0", features = ["client_requests"] } + +[features] +default = ["enum", "runtime", "closure"] +enum = ["ext-php-rs/enum"] +anyhow = ["ext-php-rs/anyhow"] +runtime = ["ext-php-rs/runtime"] +closure = ["ext-php-rs/closure"] +static = ["ext-php-rs/static"] + +[lib] +crate-type = ["cdylib"] + +[[bench]] +name = "binary_bench" +harness = false diff --git a/benches/benches/binary_bench.rs b/benches/benches/binary_bench.rs new file mode 100644 index 000000000..5502afbc7 --- /dev/null +++ b/benches/benches/binary_bench.rs @@ -0,0 +1,84 @@ +use std::{ + process::Command, + sync::{LazyLock, Once}, +}; + +use gungraun::{ + BinaryBenchmarkConfig, Callgrind, FlamegraphConfig, binary_benchmark, binary_benchmark_group, + main, +}; + +static BUILD: Once = Once::new(); +static EXT_TARGET_DIR: LazyLock = LazyLock::new(|| { + let mut dir = std::env::current_dir().expect("Could not get cwd"); + dir.pop(); + dir.push("target"); + dir.push("release"); + dir.display().to_string() +}); + +fn setup() { + BUILD.call_once(|| { + let mut command = Command::new("cargo"); + command.arg("build"); + + command.arg("--release"); + + // Build features list dynamically based on compiled features + // Note: Using vec_init_then_push pattern here is intentional due to conditional compilation + #[allow(clippy::vec_init_then_push)] + { + let mut features = vec![]; + #[cfg(feature = "enum")] + features.push("enum"); + #[cfg(feature = "closure")] + features.push("closure"); + #[cfg(feature = "anyhow")] + features.push("anyhow"); + #[cfg(feature = "runtime")] + features.push("runtime"); + #[cfg(feature = "static")] + features.push("static"); + + if !features.is_empty() { + command.arg("--no-default-features"); + command.arg("--features").arg(features.join(",")); + } + } + + let result = command.output().expect("failed to execute cargo build"); + + assert!( + result.status.success(), + "Extension build failed:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + }); +} + +#[binary_benchmark] +#[bench::single_function_call(args = ("benches/function_call.php", 1))] +#[bench::multiple_function_calls(args = ("benches/function_call.php", 10))] +#[bench::lots_of_function_calls(args = ("benches/function_call.php", 100_000))] +fn function_calls(path: &str, cnt: usize) -> gungraun::Command { + setup(); + + gungraun::Command::new("php") + .arg(format!("-dextension={}/libbenches.so", *EXT_TARGET_DIR)) + .arg(path) + .arg(cnt.to_string()) + .build() +} + +binary_benchmark_group!( + name = function; + benchmarks = function_calls +); + +main!( + config = BinaryBenchmarkConfig::default() + .tool(Callgrind::with_args(["--instr-atstart=no", "--I1=32768,8,64", "--D1=32768,8,64", "--LL=67108864,16,64"]) + .flamegraph(FlamegraphConfig::default())); + binary_benchmark_groups = function +); diff --git a/benches/benches/function_call.php b/benches/benches/function_call.php new file mode 100644 index 000000000..3c54f6b6d --- /dev/null +++ b/benches/benches/function_call.php @@ -0,0 +1,11 @@ + u64 { + // A simple function that does not do much work + n +} + +#[php_function] +pub fn bench_callback_function(callback: ZendCallable) -> Zval { + // Call the provided PHP callable with a fixed argument + callback + .try_call(vec![&42]) + .expect("Failed to call function") +} + +#[php_function] +pub fn start_instrumentation() { + gungraun::client_requests::callgrind::start_instrumentation(); + // gungraun::client_requests::callgrind::toggle_collect(); +} + +#[php_function] +pub fn stop_instrumentation() { + gungraun::client_requests::callgrind::stop_instrumentation(); +} + +#[php_module] +pub fn build_module(module: ModuleBuilder) -> ModuleBuilder { + module + .function(wrap_function!(bench_function)) + .function(wrap_function!(bench_callback_function)) + .function(wrap_function!(start_instrumentation)) + .function(wrap_function!(stop_instrumentation)) +} diff --git a/flake.lock b/flake.lock index 594401cd2..bc46cd1f5 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1762977756, - "narHash": "sha256-4PqRErxfe+2toFJFgcRKZ0UI9NSIOJa+7RXVtBhy4KE=", + "lastModified": 1765186076, + "narHash": "sha256-hM20uyap1a0M9d344I692r+ik4gTMyj60cQWO+hAYP8=", "owner": "nixos", "repo": "nixpkgs", - "rev": "c5ae371f1a6a7fd27823bc500d9390b38c05fa55", + "rev": "addf7cf5f383a3101ecfba091b98d0a1263dc9b8", "type": "github" }, "original": { @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1763087910, - "narHash": "sha256-eB9Z1mWd1U6N61+F8qwDggX0ihM55s4E0CluwNukJRU=", + "lastModified": 1765248027, + "narHash": "sha256-ngar+yP06x3+2k2Iey29uU0DWx5ur06h3iPBQXlU+yI=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "cf4a68749733d45c0420726596367acd708eb2e8", + "rev": "7b50ad68415ae5be7ee4cc68fa570c420741b644", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 3d9f93a15..e9396afa2 100644 --- a/flake.nix +++ b/flake.nix @@ -28,12 +28,14 @@ php-dev libclang.lib clang + valgrind ]; nativeBuildInputs = [ pkgs.rust-bin.stable.latest.default ]; shellHook = '' export LIBCLANG_PATH="${pkgs.libclang.lib}/lib" + export GUNGRAUN_VALGRIND_INCLUDE="${pkgs.valgrind.dev}/include" ''; }; }; From 60971aa7ee66d6f7a5f7610f162b713eebd4a7f0 Mon Sep 17 00:00:00 2001 From: Xenira <1288524+Xenira@users.noreply.github.com> Date: Mon, 15 Dec 2025 16:06:42 +0100 Subject: [PATCH 2/4] ci(bench): add benchmark tracking ci Refs: #599 --- .cargo/config.toml | 3 -- .github/workflows/master_benchmarks.yml | 34 ++++++++++++ .github/workflows/pr_benchmarks_archive.yml | 18 +++++++ .github/workflows/pr_benchmarks_run.yml | 34 ++++++++++++ .github/workflows/pr_benchmarks_upload.yml | 57 +++++++++++++++++++++ Cargo.toml | 3 +- benches/.cargo/config.toml | 14 +++++ benches/.gitignore | 1 + benches/Cargo.toml | 1 + benches/benches/binary_bench.rs | 5 +- 10 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/master_benchmarks.yml create mode 100644 .github/workflows/pr_benchmarks_archive.yml create mode 100644 .github/workflows/pr_benchmarks_run.yml create mode 100644 .github/workflows/pr_benchmarks_upload.yml create mode 100644 benches/.cargo/config.toml create mode 100644 benches/.gitignore diff --git a/.cargo/config.toml b/.cargo/config.toml index 88c4308c0..288e4ef9b 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -9,6 +9,3 @@ linker = "rust-lld" [target.'cfg(target_env = "musl")'] rustflags = ["-C", "target-feature=-crt-static"] - -[profile.bench] -debug = true diff --git a/.github/workflows/master_benchmarks.yml b/.github/workflows/master_benchmarks.yml new file mode 100644 index 000000000..fb16d58cc --- /dev/null +++ b/.github/workflows/master_benchmarks.yml @@ -0,0 +1,34 @@ +on: + push: + branches: main + +jobs: + run_benchmarks: + name: Run PR Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Valgrind + run: sudo apt update && sudo apt install -y valgrind + - uses: taiki-e/install-action@cargo-binstall + - name: Install gungraun-runner + run: | + version=$(cargo metadata --format-version=1 |\ + jq '.packages[] | select(.name == "gungraun").version' |\ + tr -d '"' + ) + cargo binstall --no-confirm gungraun-runner --version $version + - name: Run Benchmarks + run: cd ./benches && cargo bench -- --output-format json | jq -s > benchmark_results.json + - name: Track Benchmarks with Bencher + run: | + bencher run \ + --host 'https://bencher.php.rs/api' \ + --project ext-php-rs \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch master \ + --testbed ubuntu-latest \ + --err \ + --adapter rust_gungraun \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --file "./benches/benchmark_results.json" diff --git a/.github/workflows/pr_benchmarks_archive.yml b/.github/workflows/pr_benchmarks_archive.yml new file mode 100644 index 000000000..338011423 --- /dev/null +++ b/.github/workflows/pr_benchmarks_archive.yml @@ -0,0 +1,18 @@ +on: + pull_request_target: + types: [closed] + +jobs: + archive_fork_pr_branch: + name: Archive closed PR branch with Bencher + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: bencherdev/bencher@main + - name: Archive closed fork PR branch with Bencher + run: | + bencher archive \ + --host 'https://bencher.php.rs/api' \ + --project ext-php-rs \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch "$GITHUB_HEAD_REF" diff --git a/.github/workflows/pr_benchmarks_run.yml b/.github/workflows/pr_benchmarks_run.yml new file mode 100644 index 000000000..0246c5f8c --- /dev/null +++ b/.github/workflows/pr_benchmarks_run.yml @@ -0,0 +1,34 @@ +name: Run Benchmarks + +on: + pull_request: + types: [opened, reopened, edited, synchronize] + +jobs: + run_benchmarks: + name: Run PR Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Valgrind + run: sudo apt update && sudo apt install -y valgrind + - uses: taiki-e/install-action@cargo-binstall + - name: Install gungraun-runner + run: | + version=$(cargo metadata --manifest-path ./benches/Cargo.toml --format-version=1 |\ + jq '.packages[] | select(.name == "gungraun").version' |\ + tr -d '"' + ) + cargo binstall --no-confirm gungraun-runner --version $version + - name: Run Benchmarks + run: cargo bench --manifest-path ./benches/Cargo.toml -- --output-format json | jq -s > benchmark_results.json + - name: Upload Benchmark Results + uses: actions/upload-artifact@v4 + with: + name: benchmark_results.json + path: ./benchmark_results.json + - name: Upload Pull Request Event + uses: actions/upload-artifact@v4 + with: + name: event.json + path: ${{ github.event_path }} diff --git a/.github/workflows/pr_benchmarks_upload.yml b/.github/workflows/pr_benchmarks_upload.yml new file mode 100644 index 000000000..2691a9a8e --- /dev/null +++ b/.github/workflows/pr_benchmarks_upload.yml @@ -0,0 +1,57 @@ +name: Upload PR Benchmark Results + +on: + workflow_run: + workflows: [Run Benchmarks] + types: [completed] + +jobs: + upload_benchmarks: + if: github.event.workflow_run.conclusion == 'success' + permissions: + pull-requests: write + runs-on: ubuntu-latest + env: + BENCHMARK_RESULTS: benchmark_results.json + PR_EVENT: event.json + steps: + - name: Download Benchmark Results + uses: dawidd6/action-download-artifact@v6 + with: + name: ${{ env.BENCHMARK_RESULTS }} + run_id: ${{ github.event.workflow_run.id }} + - name: Download PR Event + uses: dawidd6/action-download-artifact@v6 + with: + name: ${{ env.PR_EVENT }} + run_id: ${{ github.event.workflow_run.id }} + - name: Export PR Event Data + uses: actions/github-script@v6 + with: + script: | + let fs = require('fs'); + let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'})); + core.exportVariable("PR_HEAD", prEvent.pull_request.head.ref); + core.exportVariable("PR_HEAD_SHA", prEvent.pull_request.head.sha); + core.exportVariable("PR_BASE", prEvent.pull_request.base.ref); + core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha); + core.exportVariable("PR_NUMBER", prEvent.number); + - uses: bencherdev/bencher@main + - name: Track Benchmarks with Bencher + run: | + bencher run \ + --host 'https://bencher.php.rs/api' \ + --project ext-php-rs \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch "$PR_HEAD" \ + --hash "$PR_HEAD_SHA" \ + --start-point "$PR_BASE" \ + --start-point-hash "$PR_BASE_SHA" \ + --start-point-clone-thresholds \ + --start-point-reset \ + --testbed ubuntu-latest \ + --err \ + --adapter rust_gungraun \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-number "$PR_NUMBER" \ + --file "$BENCHMARK_RESULTS" diff --git a/Cargo.toml b/Cargo.toml index 94d2e34b6..e2d49412a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,8 @@ runtime = ["bindgen/runtime"] static = ["bindgen/static"] [workspace] -members = ["crates/macros", "crates/cli", "tests", "benches"] +members = ["crates/macros", "crates/cli", "tests"] +exclude = ["benches"] [package.metadata.docs.rs] rustdoc-args = ["--cfg", "docs"] diff --git a/benches/.cargo/config.toml b/benches/.cargo/config.toml new file mode 100644 index 000000000..88c4308c0 --- /dev/null +++ b/benches/.cargo/config.toml @@ -0,0 +1,14 @@ +[target.'cfg(not(target_os = "windows"))'] +rustflags = ["-C", "link-arg=-Wl,-undefined,dynamic_lookup"] + +[target.x86_64-pc-windows-msvc] +linker = "rust-lld" + +[target.i686-pc-windows-msvc] +linker = "rust-lld" + +[target.'cfg(target_env = "musl")'] +rustflags = ["-C", "target-feature=-crt-static"] + +[profile.bench] +debug = true diff --git a/benches/.gitignore b/benches/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/benches/.gitignore @@ -0,0 +1 @@ +/target diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 22be4d28e..61deab0f3 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -18,6 +18,7 @@ static = ["ext-php-rs/static"] [lib] crate-type = ["cdylib"] +bench = false [[bench]] name = "binary_bench" diff --git a/benches/benches/binary_bench.rs b/benches/benches/binary_bench.rs index 5502afbc7..3b27f6f35 100644 --- a/benches/benches/binary_bench.rs +++ b/benches/benches/binary_bench.rs @@ -4,14 +4,13 @@ use std::{ }; use gungraun::{ - BinaryBenchmarkConfig, Callgrind, FlamegraphConfig, binary_benchmark, binary_benchmark_group, - main, + binary_benchmark, binary_benchmark_group, main, BinaryBenchmarkConfig, Callgrind, + FlamegraphConfig, }; static BUILD: Once = Once::new(); static EXT_TARGET_DIR: LazyLock = LazyLock::new(|| { let mut dir = std::env::current_dir().expect("Could not get cwd"); - dir.pop(); dir.push("target"); dir.push("release"); dir.display().to_string() From 4817cbd344e000189222fb0926539c85fe040c68 Mon Sep 17 00:00:00 2001 From: Xenira <1288524+Xenira@users.noreply.github.com> Date: Mon, 15 Dec 2025 21:10:58 +0100 Subject: [PATCH 3/4] test(benches): add callback benchmark Refs: #599 --- benches/benches/binary_bench.rs | 21 ++++++++++++++++++++- benches/benches/callback_call.php | 5 +++++ benches/src/lib.rs | 12 ++++++++---- 3 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 benches/benches/callback_call.php diff --git a/benches/benches/binary_bench.rs b/benches/benches/binary_bench.rs index 3b27f6f35..8cd81bfde 100644 --- a/benches/benches/binary_bench.rs +++ b/benches/benches/binary_bench.rs @@ -70,14 +70,33 @@ fn function_calls(path: &str, cnt: usize) -> gungraun::Command { .build() } +#[binary_benchmark] +#[bench::single_callback_call(args = ("benches/callback_call.php", 1))] +#[bench::multiple_callback_calls(args = ("benches/callback_call.php", 10))] +#[bench::lots_of_callback_calls(args = ("benches/callback_call.php", 100_000))] +fn callback_calls(path: &str, cnt: usize) -> gungraun::Command { + setup(); + + gungraun::Command::new("php") + .arg(format!("-dextension={}/libbenches.so", *EXT_TARGET_DIR)) + .arg(path) + .arg(cnt.to_string()) + .build() +} + binary_benchmark_group!( name = function; benchmarks = function_calls ); +binary_benchmark_group!( + name = callback; + benchmarks = callback_calls +); + main!( config = BinaryBenchmarkConfig::default() .tool(Callgrind::with_args(["--instr-atstart=no", "--I1=32768,8,64", "--D1=32768,8,64", "--LL=67108864,16,64"]) .flamegraph(FlamegraphConfig::default())); - binary_benchmark_groups = function + binary_benchmark_groups = function, callback ); diff --git a/benches/benches/callback_call.php b/benches/benches/callback_call.php new file mode 100644 index 000000000..04123721b --- /dev/null +++ b/benches/benches/callback_call.php @@ -0,0 +1,5 @@ + $i * 2, (int) $argv[1]); diff --git a/benches/src/lib.rs b/benches/src/lib.rs index dddcc7c45..825644f3f 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -15,11 +15,15 @@ pub fn bench_function(n: u64) -> u64 { } #[php_function] -pub fn bench_callback_function(callback: ZendCallable) -> Zval { +pub fn bench_callback_function(callback: ZendCallable, n: usize) { // Call the provided PHP callable with a fixed argument - callback - .try_call(vec![&42]) - .expect("Failed to call function") + start_instrumentation(); + for i in 0..n { + callback + .try_call(vec![&i]) + .expect("Failed to call function"); + } + stop_instrumentation(); } #[php_function] From edd7dc1f44a225b4ccd7048c96acb278e6ce5498 Mon Sep 17 00:00:00 2001 From: Xenira <1288524+Xenira@users.noreply.github.com> Date: Mon, 15 Dec 2025 22:13:44 +0100 Subject: [PATCH 4/4] ci(benches): upload correct benchmark output format Refs: #599 --- .github/workflows/master_benchmarks.yml | 6 +++--- .github/workflows/pr_benchmarks_run.yml | 6 +++--- .github/workflows/pr_benchmarks_upload.yml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/master_benchmarks.yml b/.github/workflows/master_benchmarks.yml index fb16d58cc..227913271 100644 --- a/.github/workflows/master_benchmarks.yml +++ b/.github/workflows/master_benchmarks.yml @@ -19,7 +19,7 @@ jobs: ) cargo binstall --no-confirm gungraun-runner --version $version - name: Run Benchmarks - run: cd ./benches && cargo bench -- --output-format json | jq -s > benchmark_results.json + run: cargo bench --manifest-path ./benches/Cargo.toml > benchmark_results.txt - name: Track Benchmarks with Bencher run: | bencher run \ @@ -27,8 +27,8 @@ jobs: --project ext-php-rs \ --token '${{ secrets.BENCHER_API_TOKEN }}' \ --branch master \ - --testbed ubuntu-latest \ + --testbed "$(php --version | head -n 1)" \ --err \ --adapter rust_gungraun \ --github-actions '${{ secrets.GITHUB_TOKEN }}' \ - --file "./benches/benchmark_results.json" + --file "./benchmark_results.txt" diff --git a/.github/workflows/pr_benchmarks_run.yml b/.github/workflows/pr_benchmarks_run.yml index 0246c5f8c..6da8d5ebe 100644 --- a/.github/workflows/pr_benchmarks_run.yml +++ b/.github/workflows/pr_benchmarks_run.yml @@ -21,12 +21,12 @@ jobs: ) cargo binstall --no-confirm gungraun-runner --version $version - name: Run Benchmarks - run: cargo bench --manifest-path ./benches/Cargo.toml -- --output-format json | jq -s > benchmark_results.json + run: cargo bench --manifest-path ./benches/Cargo.toml > benchmark_results.txt - name: Upload Benchmark Results uses: actions/upload-artifact@v4 with: - name: benchmark_results.json - path: ./benchmark_results.json + name: benchmark_results.txt + path: ./benchmark_results.txt - name: Upload Pull Request Event uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/pr_benchmarks_upload.yml b/.github/workflows/pr_benchmarks_upload.yml index 2691a9a8e..88acf9d79 100644 --- a/.github/workflows/pr_benchmarks_upload.yml +++ b/.github/workflows/pr_benchmarks_upload.yml @@ -12,7 +12,7 @@ jobs: pull-requests: write runs-on: ubuntu-latest env: - BENCHMARK_RESULTS: benchmark_results.json + BENCHMARK_RESULTS: benchmark_results.txt PR_EVENT: event.json steps: - name: Download Benchmark Results @@ -49,7 +49,7 @@ jobs: --start-point-hash "$PR_BASE_SHA" \ --start-point-clone-thresholds \ --start-point-reset \ - --testbed ubuntu-latest \ + --testbed "$(php --version | head -n 1)" \ --err \ --adapter rust_gungraun \ --github-actions '${{ secrets.GITHUB_TOKEN }}' \