Attributed benchmarks PoC

composer.json

@@ -23,6 +23,7 @@
     ],
     "require": {
         "php": ">=8.1",
+        "dragon-code/benchmark": "^2.6 || ^3.0",
         "internal/destroy": "^1.0",
         "internal/path": "^1.2",
         "psr/container": "1 - 2",

src/Bench/BenchWith.php

@@ -0,0 +1,47 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Testo\Bench;
+
+use Testo\Bench\Internal\BenchWithInterceptor;
+use Testo\Pipeline\Attribute\FallbackInterceptor;
+use Testo\Pipeline\Attribute\Interceptable;
+
+/**
+ * Attribute to specify additional functions to benchmark with.
+ *
+ * @api
+ */
+#[\Attribute(\Attribute::TARGET_METHOD | \Attribute::TARGET_FUNCTION | \Attribute::IS_REPEATABLE)]
+#[FallbackInterceptor(BenchWithInterceptor::class)]
+final class BenchWith implements Interceptable
+{
+    public function __construct(
+        /**
+         * @var array<callable|array{class-string, non-empty-string}> $callables Functions to benchmark with.
+         *      It might be a callable or an array with class name and non-public method name.
+         */
+        public readonly array $callables,
+
+        /**
+         * @var array Arguments to pass to the benchmarked functions.
+         */
+        public readonly array $arguments = [],
+
+        /**
+         * @var int<1, max> Number of iterations to run for each benchmark.
+         */
+        public readonly int $iterations = 1000,
+
+        /**
+         * @var int<1, max> Number of revolutions to run for each benchmark.
+         *      A revolution is a single execution of the benchmarked function.
+         */
+        public readonly int $revolutions = 5,
+    ) {
+        \count($callables) < 1 or throw new \InvalidArgumentException('At least one callable must be provided.');
+        $iterations > 0 or throw new \InvalidArgumentException('Iterations must be greater than 0.');
+        $revolutions > 0 or throw new \InvalidArgumentException('Revolutions must be greater than 0.');
+    }
+}

src/Bench/Exception/BenchWithAttributeMissingException.php

@@ -0,0 +1,34 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Testo\Bench\Exception;
+
+use Testo\Bench\BenchWith;
+use Testo\Bench\Internal\BenchWithInterceptor;
+use Testo\Core\Context\TestInfo;
+
+/**
+ * Thrown when {@see BenchWith} attribute is missing in {@see BenchInvoker}.
+ *
+ * This indicates a broken pipeline where the {@see BenchWithInterceptor} middleware
+ * did not execute as expected. The {@see BenchWith} attribute should have been set
+ * by the interceptor before reaching the invoker.
+ *
+ * @internal
+ */
+final class BenchWithAttributeMissingException extends \LogicException
+{
+    public static function fromTestInfo(TestInfo $info): self
+    {
+        return new self(
+            \sprintf(
+                'Target BenchWith attribute is missing for `%s%s()`. This indicates a broken test pipeline.',
+                $info->caseInfo->definition->reflection === null
+                    ? ''
+                    : $info->caseInfo->definition->reflection->getName() . '::',
+                $info->testDefinition->reflection->getName(),
+            ),
+        );
+    }
+}

src/Bench/Internal/BenchInvoker.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Testo\Bench\Internal;
+
+use DragonCode\Benchmark\Benchmark;
+use Testo\Bench\BenchWith;
+use Testo\Bench\Exception\BenchWithAttributeMissingException;
+use Testo\Core\Context\TestInfo;
+
+final class BenchInvoker
+{
+    private static function normalizeCallable(TestInfo $info, callable|array $callable): \Closure
+    {
+        $fn = $callable(...);
+        return static fn (): mixed => $fn(...$info->arguments);
+    }
+
+    public function __invoke(TestInfo $info): mixed
+    {
+        $attr = $info->getAttribute(BenchWith::class);
+        $attr instanceof BenchWith or throw BenchWithAttributeMissingException::fromTestInfo($info);
+
+        // Current function callable
+        $fn = $info->caseInfo->instance === null || $info->testDefinition->reflection->isStatic()
+            ? $info->testDefinition->reflection->getClosure(null)
+            : $info->testDefinition->reflection->getClosure($info->caseInfo->instance->getInstance());
+
+
+        $functions = [static fn (): mixed => $fn(...$info->arguments)];
+
+        # Collect callables
+        foreach ($attr->callables as $callable) {
+            $f = self::normalizeCallable($info, $callable);
+            $functions[] = $f;
+        }
+
+
+        Benchmark::start()
+            ->withoutData()
+            ->iterations($attr->iterations)
+            ->compare(
+                ...$functions,
+            );
+
+        return null;
+    }
+}

src/Bench/Internal/BenchWithInterceptor.php

@@ -0,0 +1,89 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Testo\Bench\Internal;
+
+use Psr\EventDispatcher\EventDispatcherInterface;
+use Testo\Bench\BenchWith;
+use Testo\Core\Context\TestInfo;
+use Testo\Core\Context\TestResult;
+use Testo\Core\Value\Status;
+use Testo\Data\MultipleResult;
+use Testo\Event\Test\TestBatchFinished;
+use Testo\Event\Test\TestBatchStarting;
+use Testo\Event\Test\TestDataSetFinished;
+use Testo\Event\Test\TestDataSetStarting;
+use Testo\Pipeline\Attribute\InterceptorOptions;
+use Testo\Pipeline\Middleware\TestRunInterceptor;
+use Testo\Pipeline\Policy\ConflictPolicy;
+
+/**
+ * Handles {@see BenchWith} attribute by running
+ *
+ * @internal
+ */
+#[InterceptorOptions(order: InterceptorOptions::ORDER_DATA_PROVIDER, onConflict: ConflictPolicy::First)]
+final class BenchWithInterceptor implements TestRunInterceptor
+{
+    public function __construct(
+        private readonly EventDispatcherInterface $eventDispatcher,
+    ) {}
+
+    #[\Override]
+    public function runTest(TestInfo $info, callable $next): TestResult
+    {
+        /** @var BenchWith[] $attributes */
+        $attributes = $info->getAttribute(BenchWith::class);
+        if ($attributes === []) {
+            return $next($info);
+        }
+
+        if (\count($attributes) === 1) {
+            $attr = \reset($attributes);
+            return $next($info->with(arguments: $attr->arguments)->withAttribute(BenchWith::class, $attr));
+        }
+
+        # Dispatch batch starting event
+        $this->eventDispatcher->dispatch(new TestBatchStarting($info));
+
+        # Run the test for each data set
+        $results = [];
+        $status = Status::Passed;
+        foreach ($attributes as $index => $attr) {
+            $newInfo = $info->with(arguments: $attr->arguments)->withAttribute(BenchWith::class, $attr);
+            $label = "$index";
+
+            # Dispatch dataset starting event
+            $this->eventDispatcher->dispatch(
+                new TestDataSetStarting($newInfo, $label, null, $index),
+            );
+
+            try {
+                $result = $next($newInfo);
+            } catch (\Throwable $throwable) {
+                $result = new TestResult(info: $newInfo, status: Status::Error, failure: $throwable);
+            }
+
+            # Dispatch dataset finished event
+            $this->eventDispatcher->dispatch(
+                new TestDataSetFinished($newInfo, $result, $label, null, $index),
+            );
+
+            unset($attr, $newInfo);
+            $result->status->isFailure() and ($status = Status::Failed);
+            $results[] = $result;
+        }
+
+        $results = new MultipleResult($results);
+
+        $finalResult = new TestResult(info: $info, status: $status, result: $results, attributes: [
+            MultipleResult::class => $results,
+        ]);
+
+        # Dispatch batch finished event
+        $this->eventDispatcher->dispatch(new TestBatchFinished($info, $finalResult));
+
+        return $finalResult;
+    }
+}

src/Bench/Middleware/BenchFinder.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Testo\Bench\Middleware;
+
+use Testo\Bench\BenchWith;
+use Testo\Bench\Internal\BenchInvoker;
+use Testo\Common\Reflection;
+use Testo\Core\Context\TestInfo;
+use Testo\Core\Definition\CaseDefinitions;
+use Testo\Pipeline\Attribute\InterceptorOptions;
+use Testo\Pipeline\Middleware\CaseLocatorInterceptor;
+use Testo\Pipeline\Middleware\FileLocatorInterceptor;
+use Testo\Tokenizer\Reflection\FileDefinitions;
+use Testo\Tokenizer\Reflection\TokenizedFile;
+
+/**
+ * Finds benchmarks defined with the {@see BenchWith} attribute.
+ */
+#[InterceptorOptions(order: -20_000)]
+final class BenchFinder implements FileLocatorInterceptor, CaseLocatorInterceptor
+{
+    /** @var \Closure(TestInfo): mixed Invoker for the test method. */
+    private readonly \Closure $invoker;
+
+    public function __construct(BenchInvoker $invoker)
+    {
+        $this->invoker = $invoker(...);
+    }
+
+    #[\Override]
+    public function locateFile(TokenizedFile $file, callable $next): ?bool
+    {
+        return $file->getClasses() !== [] || $file->getFunctions() !== [] ? true : $next($file);
+    }
+
+    #[\Override]
+    public function locateTestCases(FileDefinitions $file, callable $next): CaseDefinitions
+    {
+        // Define cases for classes
+        foreach ($file->classes as $class) {
+            if ($class->isAbstract()) {
+                continue;
+            }
+
+            $case = null;
+            foreach ($class->getMethods() as $method) {
+                if (Reflection::fetchFunctionAttributes($method, attributeClass: BenchWith::class)) {
+                    if ($case === null) {
+                        $case = $file->cases->define($class, $file);
+                        $case->invoker = $this->invoker;
+                    }
+
+                    $case->tests->define($method);
+                }
+            }
+        }
+
+        if ($file->functions === []) {
+            return $next($file);
+        }
+
+        // Define a case for functions
+        // Implement a lazy case definition
+        $case = null;
+        foreach ($file->functions as $function) {
+            if (Reflection::fetchFunctionAttributes($function, attributeClass: BenchWith::class)) {
+                if ($case === null) {
+                    $case = $file->cases->define(null, $file);
+                    $case->invoker = $this->invoker;
+                }
+
+                $case->tests->define($function);
+            }
+        }
+
+        return $next($file);
+    }
+}

src/Pipeline/InterceptorProvider.php

@@ -10,6 +10,7 @@
 use Testo\Application\Middleware\Locator\TestoAttributesLocatorInterceptor;
 use Testo\Assert\Interceptor\AssertCollectorInterceptor;
 use Testo\Assert\Interceptor\ExpectationsInterceptor;
+use Testo\Bench\Middleware\BenchFinder;
 use Testo\Common\Container;
 use Testo\Common\Reflection;
 use Testo\Inline\Middleware\TestInlineFinder;
@@ -56,6 +57,7 @@ public function fromConfig(string $class): array
         return $this->fromClasses($class, ...[
             FilterInterceptor::class,
             TestInlineFinder::class,
+            BenchFinder::class,
             new FilePostfixTestLocatorInterceptor(),
             new TestoAttributesLocatorInterceptor(),
             new AssertCollectorInterceptor(),

tests/Bench/Self/BenchWithAttr.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Bench\Self;
+
+use Testo\Bench\BenchWith;
+
+final class BenchWithAttr
+{
+    #[BenchWith(
+        [
+            [self::class, 'sumFast'],
+            [self::class, 'sumSlow'],
+        ],
+        arguments: [1, 2],
+        iterations: 10_000,
+    )]
+    public static function sumFast(int $a, int $b): int
+    {
+        return $a + $b;
+    }
+
+    public static function sumSlow(int $a, int $b): int
+    {
+        return (int) \array_sum([$a, $b]);
+    }
+}

tests/Bench/suites.php

@@ -0,0 +1,15 @@
+<?php
+
+declare(strict_types=1);
+
+use Testo\Application\Config\FinderConfig;
+use Testo\Application\Config\SuiteConfig;
+
+return [
+    new SuiteConfig(
+        name: 'Bench/Self',
+        location: new FinderConfig(
+            include: [__DIR__ . '/Self'],
+        ),
+    ),
+];