From d7479d1929cfe286a055931187e67c6af4a37a33 Mon Sep 17 00:00:00 2001 From: Naseem Alnaji Date: Thu, 22 Oct 2026 20:00:03 +0000 Subject: [PATCH 1/2] ci: test the newest patch of each MCP library minor release from Packagist CI tested fixed versions of the MCP libraries: the lowest and highest of each require-dev range, plus mcp/sdk 0.9.0 pinned. A caret range on 0.x stops at the next minor, so a new mcp/sdk 0.9.1 or 0.10, symfony/mcp-bundle 0.15 or laravel/mcp 2.0 never ran, although composer.json lets a customer install it. Like the TypeScript SDK's mcp-compatibility.yml, a discover job now reads Packagist on every pull request and every night. It picks the newest patch of each minor release of mcp/sdk, laravel/mcp and symfony/mcp-bundle, from the lowest version that require-dev allows. The released jobs run PHPStan and the adapter suites against each of them, and the package probe runs on each mcp/sdk release. In the nightly, a failing release opens an issue. - tests/Support/ReleaseDiscovery.php selects the releases, and tests/bin/released-versions.php prints them for the matrix. - tests/bin/require-library.sh installs one version of a library. The local tests/bin/mcp-sdk-leg.sh uses it too. - tests/Support/HttpGet.php holds the GET that branding-drift.php had inline. - WorkflowsTest makes sure that All green needs every gating CI job, that the released jobs take the discovered releases, and that every nightly job opens its issue when it fails. - The released jobs replace the mcp/sdk 0.9.0 job. --- .github/workflows/ci.yml | 52 +++++++-- .github/workflows/nightly.yml | 85 ++++++++++++++ CONTRIBUTING.md | 5 +- README.md | 2 +- tests/Architecture/WorkflowsTest.php | 131 +++++++++++++++++++++ tests/Support/HttpGet.php | 38 ++++++ tests/Support/ReleaseDiscovery.php | 130 +++++++++++++++++++++ tests/Unit/ReleaseDiscoveryTest.php | 169 +++++++++++++++++++++++++++ tests/bin/branding-drift.php | 30 +---- tests/bin/mcp-sdk-leg.sh | 5 +- tests/bin/released-versions.php | 37 ++++++ tests/bin/require-library.sh | 27 +++++ 12 files changed, 670 insertions(+), 41 deletions(-) create mode 100644 tests/Architecture/WorkflowsTest.php create mode 100644 tests/Support/HttpGet.php create mode 100644 tests/Support/ReleaseDiscovery.php create mode 100644 tests/Unit/ReleaseDiscoveryTest.php create mode 100644 tests/bin/released-versions.php create mode 100755 tests/bin/require-library.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53d341e..ec081a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,10 +159,42 @@ jobs: if: github.event_name != 'pull_request' run: vendor/bin/infection --threads=max --logger-github - mcp-sdk-0-9: - name: mcp/sdk 0.9.0 (PHP 8.5) + discover: + name: Discover the MCP library releases (Packagist) + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + releases: ${{ steps.releases.outputs.releases }} + mcp-sdk: ${{ steps.releases.outputs.mcp-sdk }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # 2.40.0 + with: + php-version: '8.5' + coverage: none + - uses: ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda # 4.0.0 + with: + dependency-versions: highest + # The newest patch of each minor release from the require-dev floor, so a new release joins the matrix by itself. + - name: Read the releases from Packagist + id: releases + run: | + releases="$(php tests/bin/released-versions.php)" + echo "releases=${releases}" >>"$GITHUB_OUTPUT" + echo "mcp-sdk=$(jq -c 'map(select(.library == "mcp/sdk") | .version)' <<<"$releases")" >>"$GITHUB_OUTPUT" + jq -r '.[] | "- \(.library) \(.version)"' <<<"$releases" >>"$GITHUB_STEP_SUMMARY" + + released: + name: ${{ matrix.library }} ${{ matrix.version }} (PHP 8.5) + needs: discover runs-on: ubuntu-latest timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.discover.outputs.releases) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -174,11 +206,12 @@ jobs: - uses: ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda # 4.0.0 with: dependency-versions: highest - - name: Remove symfony/mcp-bundle, which requires mcp/sdk ^0.8.1 - run: composer remove --dev --no-update symfony/mcp-bundle - - name: Install the mcp/sdk 0.9.0 tag - run: composer require --dev --no-interaction --update-with-all-dependencies "mcp/sdk:0.9.0" - - run: vendor/bin/phpstan analyse --no-progress --error-format=github --configuration=phpstan.no-bundle.neon + - name: Install ${{ matrix.library }} ${{ matrix.version }} + env: + LIBRARY: ${{ matrix.library }} + VERSION: ${{ matrix.version }} + run: tests/bin/require-library.sh "$LIBRARY" "$VERSION" + - run: vendor/bin/phpstan analyse --no-progress --error-format=github --memory-limit=1G --configuration=${{ matrix.library == 'mcp/sdk' && 'phpstan.no-bundle.neon' || 'phpstan.neon.dist' }} - run: vendor/bin/phpunit --testsuite Contract,Integration,E2E runtime: @@ -207,12 +240,13 @@ jobs: package: name: Package probe (PHP 8.3, mcp/sdk ${{ matrix.mcp-sdk }}) + needs: discover runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: - mcp-sdk: ['0.8.1', '0.9.0'] + mcp-sdk: ${{ fromJSON(needs.discover.outputs.mcp-sdk) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -250,7 +284,7 @@ jobs: all-green: name: All green if: always() - needs: [static, static-lowest, tests, coverage, mutation, mcp-sdk-0-9, runtime, package, bc] + needs: [static, static-lowest, tests, coverage, mutation, discover, released, runtime, package, bc] runs-on: ubuntu-latest timeout-minutes: 5 steps: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 83434da..75e627f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -150,6 +150,91 @@ jobs: if [ -s "$OUTPUT" ]; then { printf '~~~text\n'; cat "$OUTPUT"; printf '~~~\n'; } >"$REPORT"; fi .github/nightly-issue.sh "${{ github.job }}" "$REPORT" + discover: + name: Discover the MCP library releases (Packagist) + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + releases: ${{ steps.releases.outputs.releases }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # 2.40.0 + with: + php-version: '8.5' + coverage: none + - uses: ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda # 4.0.0 + with: + dependency-versions: highest + # The same releases as the released jobs of ci.yml: a release published since the last pull request runs here. + - name: Read the releases from Packagist + id: releases + timeout-minutes: 2 + run: | + releases="$(php tests/bin/released-versions.php)" + echo "releases=${releases}" >>"$GITHUB_OUTPUT" + jq -r '.[] | "- \(.library) \(.version)"' <<<"$releases" >>"$GITHUB_STEP_SUMMARY" + - name: Fail on purpose (workflow_dispatch force_failure) + if: github.event_name == 'workflow_dispatch' && inputs.force_failure == github.job + run: exit 1 + - name: Open or update the nightly issue + if: failure() && github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + run: .github/nightly-issue.sh "${{ github.job }}" + + released: + name: ${{ matrix.library }} ${{ matrix.version }} (PHP 8.5) + needs: discover + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.discover.outputs.releases) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # 2.40.0 + with: + php-version: '8.5' + coverage: none + - uses: ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda # 4.0.0 + with: + dependency-versions: highest + - name: Install ${{ matrix.library }} ${{ matrix.version }} + timeout-minutes: 8 + env: + LIBRARY: ${{ matrix.library }} + VERSION: ${{ matrix.version }} + run: tests/bin/require-library.sh "$LIBRARY" "$VERSION" + - run: vendor/bin/phpstan analyse --no-progress --error-format=github --memory-limit=1G --configuration=${{ matrix.library == 'mcp/sdk' && 'phpstan.no-bundle.neon' || 'phpstan.neon.dist' }} + timeout-minutes: 6 + - name: Run the tests + timeout-minutes: 12 + env: + OUTPUT: ${{ runner.temp }}/phpunit.txt + run: | + set -o pipefail + vendor/bin/phpunit --testsuite Contract,Integration,E2E --no-progress --colors=never 2>&1 | tee "$OUTPUT" + - name: Fail on purpose (workflow_dispatch force_failure) + if: github.event_name == 'workflow_dispatch' && inputs.force_failure == github.job + run: exit 1 + - name: Open or update the nightly issue + if: failure() && github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + LIBRARY: ${{ matrix.library }} + VERSION: ${{ matrix.version }} + OUTPUT: ${{ runner.temp }}/phpunit.txt + REPORT: ${{ runner.temp }}/phpunit.md + run: | + # The failing tests and their messages, STDERR included, when PHPUnit ran: the issue names the failing test. + if [ -s "$OUTPUT" ]; then { printf '~~~text\n'; cat "$OUTPUT"; printf '~~~\n'; } >"$REPORT"; fi + .github/nightly-issue.sh "released (${LIBRARY} ${VERSION})" "$REPORT" + php-nightly: name: Tests (PHP nightly) runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d9a238f..c00056d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,7 +90,7 @@ git commit -m "docs: update API documentation" ### The Required Check -The one required check is `All green`. It passes when every CI job passes: static analysis on the lowest and the highest dependencies, the tests on PHP 8.3, 8.4 and 8.5, coverage, mutation testing, the `mcp/sdk` 0.9.0 leg, the runtime suite, the package probe and the backward-compatibility check. +The one required check is `All green`. It passes when every CI job passes: static analysis on the lowest and the highest dependencies, the tests on PHP 8.3, 8.4 and 8.5, coverage, mutation testing, the released-version legs, the runtime suite, the package probe and the backward-compatibility check. The released-version legs and the package probe take their versions from Packagist on each run: the newest patch of each minor release of `mcp/sdk`, `laravel/mcp` and `symfony/mcp-bundle`, from the lowest version that `require-dev` allows. The nightly runs the same legs, so a release published between pull requests opens an issue. ### No Issue Required @@ -117,6 +117,9 @@ vendor/bin/infection --threads=max --filter=src/Internal/Engine/ToolListing.php # PHPStan and the adapter suites against mcp/sdk 0.9.0, in a throwaway copy under .cache/ tests/bin/mcp-sdk-leg.sh +# The releases that the released-version legs test, read from Packagist +php tests/bin/released-versions.php + # The runtime suite: PHP-FPM, FrankenPHP, Apache, NGINX Unit, the CLI and Swoole, in Docker AGENTCAT_RUNTIME_TESTS=1 vendor/bin/phpunit --testsuite Runtime --group runtime diff --git a/README.md b/README.md index 0abbbc0..2726265 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ Diagnostics are on by default and can be turned off completely with either: - **Containers**: start a stdio server with `docker run --init`, so that a stop signal ends PHP and AgentCat (with ext-pcntl) first delivers the queued events. - **Swoole, RoadRunner and Workerman**: with Laravel or Symfony, AgentCat delivers in the framework's terminate hooks (Laravel's `terminating()` callbacks, also under Octane, and Symfony's `kernel.terminate`), waiting at most 1 s after each request; a plain `mcp/sdk` worker calls `AgentCat::flush(1.0)` after each request. -AgentCat supports PHP 8.3, 8.4 and 8.5; `mcp/sdk` 0.8.1 or later (tested 0.8.1 and 0.9.0); `laravel/mcp` 1.x (tested 1.0.0 and 1.1.0) with Laravel 12.41.1 or later and 13; and `symfony/mcp-bundle` 0.13 and 0.14 with Symfony 7.3 and 8.x. The [runtimes guide](https://docs.agentcat.com/sdk/php/runtimes) covers each runtime, containers and hardened hosts, and the [limits page](https://docs.agentcat.com/sdk/php/limits) lists the differences from the TypeScript SDK and the known limits. +AgentCat supports PHP 8.3, 8.4 and 8.5; `mcp/sdk` 0.8.1 or later; `laravel/mcp` 1.x with Laravel 12.41.1 or later and 13; and `symfony/mcp-bundle` 0.13 and 0.14 with Symfony 7.3 and 8.x. On every pull request and every night, CI reads Packagist and tests the newest patch of each minor release of these three libraries, from the lowest supported version up. The [runtimes guide](https://docs.agentcat.com/sdk/php/runtimes) covers each runtime, containers and hardened hosts, and the [limits page](https://docs.agentcat.com/sdk/php/limits) lists the differences from the TypeScript SDK and the known limits. ## Free for open source diff --git a/tests/Architecture/WorkflowsTest.php b/tests/Architecture/WorkflowsTest.php new file mode 100644 index 0000000..f197f44 --- /dev/null +++ b/tests/Architecture/WorkflowsTest.php @@ -0,0 +1,131 @@ +jobs('ci.yml'); + $gating = []; + foreach ($jobs as $id => $job) { + if ($id !== 'all-green' && ($job['continue-on-error'] ?? false) !== true) { + $gating[] = $id; + } + } + $needs = $jobs['all-green']['needs'] ?? null; + $this->assertIsArray($needs); + sort($gating); + sort($needs); + + $this->assertSame($gating, $needs, 'All green needs every CI job but the experimental ones'); + $this->assertContains('released', $needs); + } + + /** @return iterable */ + public static function workflows(): iterable + { + yield 'CI' => ['ci.yml']; + yield 'the nightly' => ['nightly.yml']; + } + + #[DataProvider('workflows')] + public function testTheReleasedJobsTestTheDiscoveredReleases(string $workflow): void + { + $jobs = $this->jobs($workflow); + + $this->assertStringContainsString('php tests/bin/released-versions.php', $this->runs($jobs['discover'] ?? [])); + $this->assertSame('${{ steps.releases.outputs.releases }}', $this->at($jobs, 'discover', 'outputs', 'releases')); + $this->assertSame('discover', $this->at($jobs, 'released', 'needs')); + $this->assertSame('${{ fromJSON(needs.discover.outputs.releases) }}', $this->at($jobs, 'released', 'strategy', 'matrix', 'include')); + $this->assertFalse($this->at($jobs, 'released', 'strategy', 'fail-fast'), 'one failing release does not cancel the others'); + $this->assertStringContainsString('tests/bin/require-library.sh "$LIBRARY" "$VERSION"', $this->runs($jobs['released'] ?? [])); + $this->assertStringContainsString('vendor/bin/phpunit --testsuite Contract,Integration,E2E', $this->runs($jobs['released'] ?? [])); + } + + public function testThePackageProbeRunsOnEachDiscoveredMcpSdkRelease(): void + { + $jobs = $this->jobs('ci.yml'); + + $this->assertSame('discover', $this->at($jobs, 'package', 'needs')); + $this->assertSame('${{ fromJSON(needs.discover.outputs.mcp-sdk) }}', $this->at($jobs, 'package', 'strategy', 'matrix', 'mcp-sdk')); + } + + /** A scheduled failure reaches a person: every nightly job ends by opening or updating its issue. */ + public function testEveryNightlyJobOpensTheIssueWhenItFails(): void + { + foreach ($this->jobs('nightly.yml') as $id => $job) { + $steps = $job['steps'] ?? []; + $this->assertIsArray($steps); + $last = end($steps); + $this->assertIsArray($last, $id); + $this->assertSame(self::ISSUE_STEP, $last['name'] ?? null, $id); + $this->assertSame("failure() && github.event_name != 'pull_request'", $last['if'] ?? null, $id); + $run = $last['run'] ?? null; + $this->assertIsString($run, $id); + $this->assertStringContainsString('.github/nightly-issue.sh', $run, $id); + } + } + + /** @return array> */ + private function jobs(string $workflow): array + { + $parsed = Yaml::parseFile(self::WORKFLOWS . '/' . $workflow); + $this->assertIsArray($parsed); + $jobs = $parsed['jobs'] ?? null; + $this->assertIsArray($jobs); + + /** @var array> $jobs */ + return $jobs; + } + + /** + * The value at a path of keys, each level an array. + * + * @param array $value + */ + private function at(array $value, string ...$keys): mixed + { + $at = $value; + foreach ($keys as $key) { + $this->assertIsArray($at, implode('.', $keys)); + $at = $at[$key] ?? null; + } + + return $at; + } + + /** + * The `run` scripts of a job's steps, one after the other. + * + * @param array $job + */ + private function runs(array $job): string + { + $steps = $job['steps'] ?? []; + $this->assertIsArray($steps); + $runs = ''; + foreach ($steps as $step) { + if (is_array($step) && is_string($step['run'] ?? null)) { + $runs .= $step['run'] . "\n"; + } + } + + return $runs; + } +} diff --git a/tests/Support/HttpGet.php b/tests/Support/HttpGet.php new file mode 100644 index 0000000..f315ee2 --- /dev/null +++ b/tests/Support/HttpGet.php @@ -0,0 +1,38 @@ + true, + \CURLOPT_FOLLOWLOCATION => true, + \CURLOPT_MAXREDIRS => 3, + \CURLOPT_CONNECTTIMEOUT => 10, + \CURLOPT_TIMEOUT => 30, + \CURLOPT_USERAGENT => 'agentcat-php-sdk ' . $script, + \CURLOPT_HTTPHEADER => array_values($headers), + ]); + $body = curl_exec($handle); + $status = curl_getinfo($handle, \CURLINFO_RESPONSE_CODE); + $error = curl_error($handle); + if (!is_string($body) || $status !== 200) { + throw new \RuntimeException(sprintf('GET %s: %s', $url, $error !== '' ? $error : 'HTTP ' . $status)); + } + + return $body; + } +} diff --git a/tests/Support/ReleaseDiscovery.php b/tests/Support/ReleaseDiscovery.php new file mode 100644 index 0000000..3965603 --- /dev/null +++ b/tests/Support/ReleaseDiscovery.php @@ -0,0 +1,130 @@ + library => version, in the order of LIBRARIES + */ + public static function floors(string $composerJson): array + { + $composer = json_decode($composerJson, true, flags: \JSON_THROW_ON_ERROR); + $requireDev = is_array($composer) && is_array($composer['require-dev'] ?? null) ? $composer['require-dev'] : []; + $floors = []; + foreach (self::LIBRARIES as $library) { + $constraint = $requireDev[$library] ?? null; + if (!is_string($constraint)) { + throw new \UnexpectedValueException('composer.json has no require-dev constraint for ' . $library); + } + $floors[$library] = self::floor($constraint); + } + + return $floors; + } + + /** The lowest version that `^` constraints joined by `||` allow, as major.minor.patch. */ + public static function floor(string $constraint): string + { + $alternatives = preg_split('/\s*\|\|?\s*/', trim($constraint)); + $floor = null; + foreach (is_array($alternatives) ? $alternatives : [] as $alternative) { + if (preg_match(self::CARET, $alternative, $parts) !== 1) { + throw self::unreadable($constraint); + } + $version = $parts[1] . '.' . $parts[2] . '.' . ($parts[3] ?? '0'); + if ($floor === null || version_compare($version, $floor, '<')) { + $floor = $version; + } + } + + return $floor ?? throw self::unreadable($constraint); + } + + /** The Packagist metadata of a library's tags; its branches are in `~dev.json`. */ + public static function metadataUrl(string $library): string + { + return 'https://repo.packagist.org/p2/' . $library . '.json'; + } + + /** + * The stable versions of a Packagist `p2` body, in its order, as major.minor.patch (a fourth number only when it is + * not 0). Packagist minifies the body, but each row keeps its own `version_normalized`. + * + * @return list + */ + public static function stableVersions(string $body, string $library): array + { + $metadata = json_decode($body, true); + if (!is_array($metadata)) { + throw new \UnexpectedValueException('the Packagist body of ' . $library . ' is not JSON'); + } + $rows = is_array($metadata['packages'] ?? null) ? $metadata['packages'][$library] ?? null : null; + if (!is_array($rows)) { + throw new \UnexpectedValueException('the Packagist body has no versions of ' . $library); + } + $versions = []; + foreach ($rows as $row) { + $normalized = is_array($row) ? $row['version_normalized'] ?? null : null; + if (!is_string($normalized)) { + throw new \UnexpectedValueException('a version of ' . $library . ' on Packagist has no version_normalized'); + } + if (preg_match(self::STABLE, $normalized, $parts) === 1) { + $versions[] = $parts[1] . '.' . $parts[2] . '.' . $parts[3] . ($parts[4] === '0' ? '' : '.' . $parts[4]); + } + } + + return $versions; + } + + /** + * The newest patch of each minor version at or above the floor, oldest first. + * + * @param list $versions + * + * @return list + */ + public static function newestPatches(array $versions, string $floor): array + { + $newest = []; + foreach ($versions as $version) { + if (version_compare($version, $floor, '<')) { + continue; + } + $minor = implode('.', array_slice(explode('.', $version), 0, 2)); + if (!isset($newest[$minor]) || version_compare($version, $newest[$minor], '>')) { + $newest[$minor] = $version; + } + } + $patches = array_values($newest); + // Each part is digits only, so the natural order is the version order: 0.9.0 before 0.10.0. + sort($patches, \SORT_NATURAL); + + return $patches; + } + + private static function unreadable(string $constraint): \InvalidArgumentException + { + return new \InvalidArgumentException('release discovery reads only ^ constraints joined by ||, got "' . $constraint . '"'); + } +} diff --git a/tests/Unit/ReleaseDiscoveryTest.php b/tests/Unit/ReleaseDiscoveryTest.php new file mode 100644 index 0000000..383a401 --- /dev/null +++ b/tests/Unit/ReleaseDiscoveryTest.php @@ -0,0 +1,169 @@ + */ + public static function constraints(): iterable + { + yield 'a caret with a patch' => ['^0.8.1', '0.8.1']; + yield 'a caret without a patch' => ['^1.0', '1.0.0']; + yield 'two carets' => ['^0.13 || ^0.14', '0.13.0']; + yield 'two carets, the newer first' => ['^0.14 || ^0.13', '0.13.0']; + yield 'one pipe' => ['^12.41.1 | ^13.0', '12.41.1']; + yield 'a two-digit minor' => ['^0.10 || ^0.9.2', '0.9.2']; + } + + #[DataProvider('constraints')] + public function testTheFloorIsTheLowestVersionTheConstraintAllows(string $constraint, string $floor): void + { + $this->assertSame($floor, ReleaseDiscovery::floor($constraint)); + } + + /** @return iterable */ + public static function otherConstraints(): iterable + { + yield 'a range' => ['>=1.0']; + yield 'a tilde' => ['~0.0.4']; + yield 'a wildcard' => ['1.*']; + yield 'a caret and a range' => ['^1.0 || >=2.0']; + yield 'nothing' => ['']; + } + + /** CI must test the floor that Composer lets a customer install, so a constraint it cannot read stops the run. */ + #[DataProvider('otherConstraints')] + public function testAConstraintOtherThanCaretsIsRejected(string $constraint): void + { + $thrown = Thrown::by(\InvalidArgumentException::class, static fn(): string => ReleaseDiscovery::floor($constraint)); + + $this->assertSame('release discovery reads only ^ constraints joined by ||, got "' . $constraint . '"', $thrown->getMessage()); + } + + public function testTheFloorsComeFromRequireDev(): void + { + $composer = $this->composer([ + 'require-dev' => ['mcp/sdk' => '^0.8.1', 'laravel/mcp' => '^1.0', 'symfony/mcp-bundle' => '^0.13 || ^0.14', 'phpunit/phpunit' => '^12.5'], + ]); + + $this->assertSame(['mcp/sdk' => '0.8.1', 'laravel/mcp' => '1.0.0', 'symfony/mcp-bundle' => '0.13.0'], ReleaseDiscovery::floors($composer)); + } + + public function testALibraryMissingFromRequireDevStopsTheRun(): void + { + $composer = $this->composer(['require-dev' => ['mcp/sdk' => '^0.8.1', 'laravel/mcp' => '^1.0']]); + + $thrown = Thrown::by(\UnexpectedValueException::class, static fn(): array => ReleaseDiscovery::floors($composer)); + + $this->assertSame('composer.json has no require-dev constraint for symfony/mcp-bundle', $thrown->getMessage()); + } + + /** + * A customer can install any version at or above the `conflict` floor, so the lowest-dependencies jobs and the + * discovery must start there too. + */ + public function testTheRepositoryFloorsMatchItsConflictFloors(): void + { + $json = (string) file_get_contents(self::ROOT . '/composer.json'); + $floors = ReleaseDiscovery::floors($json); + /** @var array{conflict: array} $composer */ + $composer = json_decode($json, true, flags: \JSON_THROW_ON_ERROR); + $conflictFloors = []; + foreach (array_intersect_key($composer['conflict'], $floors) as $library => $constraint) { + $this->assertMatchesRegularExpression('/\A<\d+\.\d+(?:\.\d+)?\z/', $constraint); + $conflictFloors[$library] = ReleaseDiscovery::floor('^' . substr($constraint, 1)); + } + ksort($conflictFloors); + $expected = array_intersect_key($floors, $conflictFloors); + ksort($expected); + + $this->assertSame(['mcp/sdk', 'laravel/mcp', 'symfony/mcp-bundle'], array_keys($floors)); + $this->assertSame(['laravel/mcp', 'mcp/sdk'], array_keys($conflictFloors), 'symfony/mcp-bundle has no conflict floor'); + $this->assertSame($expected, $conflictFloors); + } + + public function testTheStableVersionsOfAPackagistBody(): void + { + $body = $this->packagist('mcp/sdk', [ + ['version' => 'v0.9.0', 'version_normalized' => '0.9.0.0', 'require' => ['php' => '^8.1']], + ['version' => 'v0.9.0-RC1', 'version_normalized' => '0.9.0.0-RC1'], + ['version' => 'v0.8.1', 'version_normalized' => '0.8.1.0'], + ['version' => '0.8.0-beta.2', 'version_normalized' => '0.8.0.0-beta2'], + ['version' => 'dev-main', 'version_normalized' => 'dev-main'], + ['version' => '1.2.3.4', 'version_normalized' => '1.2.3.4'], + ]); + + $this->assertSame(['0.9.0', '0.8.1', '1.2.3.4'], ReleaseDiscovery::stableVersions($body, 'mcp/sdk')); + } + + /** @return iterable */ + public static function unreadableBodies(): iterable + { + yield 'not JSON' => ['', 'the Packagist body of mcp/sdk is not JSON']; + yield 'another package' => ['{"packages":{"laravel/mcp":[]}}', 'the Packagist body has no versions of mcp/sdk']; + yield 'no packages' => ['{"minified":"composer/2.0"}', 'the Packagist body has no versions of mcp/sdk']; + yield 'a row without its version' => ['{"packages":{"mcp/sdk":[{"version":"v0.9.0"}]}}', 'a version of mcp/sdk on Packagist has no version_normalized']; + } + + #[DataProvider('unreadableBodies')] + public function testABodyItCannotReadStopsTheRun(string $body, string $message): void + { + $thrown = Thrown::by(\UnexpectedValueException::class, static fn(): array => ReleaseDiscovery::stableVersions($body, 'mcp/sdk')); + + $this->assertSame($message, $thrown->getMessage()); + } + + /** @return iterable, string, list}> */ + public static function releases(): iterable + { + yield 'mcp/sdk today' => [['0.9.0', '0.8.1', '0.8.0', '0.7.1', '0.7.0'], '0.8.1', ['0.8.1', '0.9.0']]; + yield 'laravel/mcp today' => [['1.1.1', '1.1.0', '1.0.2', '1.0.1', '1.0.0', '0.9.7'], '1.0.0', ['1.0.2', '1.1.1']]; + yield 'a new patch' => [['0.9.1', '0.9.0', '0.8.1'], '0.8.1', ['0.8.1', '0.9.1']]; + yield 'a new minor' => [['0.10.0', '0.9.0', '0.8.1'], '0.8.1', ['0.8.1', '0.9.0', '0.10.0']]; + yield 'a new major' => [['2.0.0', '1.1.1', '1.0.0'], '1.0.0', ['1.0.0', '1.1.1', '2.0.0']]; + yield 'a floor inside a minor' => [['0.8.2', '0.8.1', '0.8.0'], '0.8.1', ['0.8.2']]; + yield 'a patch below the floor' => [['0.8.0', '0.9.0'], '0.8.1', ['0.9.0']]; + yield 'numbers, not text' => [['10.0.0', '9.10.0', '9.9.0', '2.0.0'], '1.0.0', ['2.0.0', '9.9.0', '9.10.0', '10.0.0']]; + yield 'nothing at the floor' => [['0.7.1'], '0.8.1', []]; + } + + /** + * @param list $versions + * @param list $expected + */ + #[DataProvider('releases')] + public function testTheNewestPatchOfEachMinorAtOrAboveTheFloor(array $versions, string $floor, array $expected): void + { + $this->assertSame($expected, ReleaseDiscovery::newestPatches($versions, $floor)); + } + + public function testTheMetadataUrlHoldsTheTaggedReleases(): void + { + $this->assertSame('https://repo.packagist.org/p2/symfony/mcp-bundle.json', ReleaseDiscovery::metadataUrl('symfony/mcp-bundle')); + } + + /** @param array $composer */ + private function composer(array $composer): string + { + return json_encode($composer, \JSON_THROW_ON_ERROR); + } + + /** @param list> $versions */ + private function packagist(string $library, array $versions): string + { + return json_encode(['minified' => 'composer/2.0', 'packages' => [$library => $versions]], \JSON_THROW_ON_ERROR); + } +} diff --git a/tests/bin/branding-drift.php b/tests/bin/branding-drift.php index b64904d..a82aced 100644 --- a/tests/bin/branding-drift.php +++ b/tests/bin/branding-drift.php @@ -11,6 +11,7 @@ // docs/ and its hash. With GITHUB_TOKEN set, the GitHub API call sends it as a bearer token. use AgentCat\Tests\Support\Branding; +use AgentCat\Tests\Support\HttpGet; require __DIR__ . '/../../vendor/autoload.php'; @@ -18,31 +19,6 @@ const TREE_URL = 'https://api.github.com/repos/agentcathq/agentcat-typescript-sdk/git/trees/main?recursive=1'; const FIXTURES = __DIR__ . '/../Fixtures/branding'; -// The body of a GET that answered 200; a RuntimeException when the request fails or answers another status. -$get = static function (string $url, string ...$headers): string { - $handle = curl_init($url); - if ($handle === false) { - throw new RuntimeException('GET ' . $url . ': curl_init() failed'); - } - curl_setopt_array($handle, [ - CURLOPT_RETURNTRANSFER => true, - CURLOPT_FOLLOWLOCATION => true, - CURLOPT_MAXREDIRS => 3, - CURLOPT_CONNECTTIMEOUT => 10, - CURLOPT_TIMEOUT => 30, - CURLOPT_USERAGENT => 'agentcat-php-sdk branding-drift', - CURLOPT_HTTPHEADER => array_values($headers), - ]); - $body = curl_exec($handle); - $status = curl_getinfo($handle, CURLINFO_RESPONSE_CODE); - $error = curl_error($handle); - if (!is_string($body) || $status !== 200) { - throw new RuntimeException(sprintf('GET %s: %s', $url, $error !== '' ? $error : 'HTTP ' . $status)); - } - - return $body; -}; - try { $images = require FIXTURES . '/images.php'; if (!is_array($images)) { @@ -62,8 +38,8 @@ if (is_string($token) && $token !== '') { $apiHeaders[] = 'Authorization: Bearer ' . $token; } - $upstream = $get(README_URL); - $upstreamImages = Branding::treeHashes($get(TREE_URL, ...$apiHeaders)); + $upstream = HttpGet::body(README_URL, 'branding-drift'); + $upstreamImages = Branding::treeHashes(HttpGet::body(TREE_URL, 'branding-drift', ...$apiHeaders)); } catch (Throwable $e) { fwrite(STDERR, 'branding-drift: ' . $e->getMessage() . "\n"); exit(2); diff --git a/tests/bin/mcp-sdk-leg.sh b/tests/bin/mcp-sdk-leg.sh index 44f777d..33a4059 100755 --- a/tests/bin/mcp-sdk-leg.sh +++ b/tests/bin/mcp-sdk-leg.sh @@ -19,8 +19,7 @@ cd "$copy" if [ -n "${MCP_SDK_REPO:-}" ]; then composer config repositories.mcp-sdk-leg vcs "$MCP_SDK_REPO" fi -# symfony/mcp-bundle requires mcp/sdk ^0.8.1, which 0.9 and main do not satisfy: the Symfony tests skip without it. -composer remove --dev --no-update symfony/mcp-bundle -composer require --dev --no-interaction --update-with-all-dependencies "mcp/sdk:${constraint}" +# Removes symfony/mcp-bundle, which requires mcp/sdk ^0.8.1: the Symfony tests skip without it. +tests/bin/require-library.sh mcp/sdk "$constraint" vendor/bin/phpstan analyse --no-progress --memory-limit=1G --configuration=phpstan.no-bundle.neon vendor/bin/phpunit --testsuite "${SUITES:-Contract,Integration,E2E}" diff --git a/tests/bin/released-versions.php b/tests/bin/released-versions.php new file mode 100644 index 0000000..4c6b748 --- /dev/null +++ b/tests/bin/released-versions.php @@ -0,0 +1,37 @@ + $floor) { + $body = HttpGet::body(ReleaseDiscovery::metadataUrl($library), 'released-versions'); + $versions = ReleaseDiscovery::newestPatches(ReleaseDiscovery::stableVersions($body, $library), $floor); + if ($versions === []) { + fwrite(STDERR, sprintf("released-versions: Packagist has no release of %s at or above %s\n", $library, $floor)); + exit(1); + } + foreach ($versions as $version) { + $rows[] = ['library' => $library, 'version' => $version]; + } + } +} catch (RuntimeException $e) { + fwrite(STDERR, 'released-versions: ' . $e->getMessage() . "\n"); + exit(2); +} + +echo json_encode($rows, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), "\n"; diff --git a/tests/bin/require-library.sh b/tests/bin/require-library.sh new file mode 100755 index 0000000..97c3b50 --- /dev/null +++ b/tests/bin/require-library.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Installs one version of an MCP library over the installed dependencies, for the `released` jobs of ci.yml and +# nightly.yml (D72) and for tests/bin/mcp-sdk-leg.sh. PHPStan then reads phpstan.no-bundle.neon after an mcp/sdk +# install, which removes symfony/mcp-bundle, and phpstan.neon.dist after the others. +# Usage: tests/bin/require-library.sh (for example 0.9.0) +set -euo pipefail +library="${1:?usage: require-library.sh }" +constraint="${2:?usage: require-library.sh }" +case "$library" in + mcp/sdk) + # symfony/mcp-bundle requires mcp/sdk ^0.8.1, which 0.9 and main do not satisfy: the Symfony tests skip without + # it, and the bundle's own rows test it. + composer remove --dev --no-update symfony/mcp-bundle + composer require --dev --no-interaction --update-with-all-dependencies "mcp/sdk:${constraint}" + ;; + laravel/mcp) + composer require --dev --no-interaction --update-with-all-dependencies "laravel/mcp:${constraint}" + ;; + symfony/mcp-bundle) + # The bundle chooses its mcp/sdk, so a release that needs a newer mcp/sdk than require-dev allows still installs. + composer require --dev --no-interaction --update-with-all-dependencies "symfony/mcp-bundle:${constraint}" "mcp/sdk:*" + ;; + *) + echo "require-library.sh: unknown library ${library}" >&2 + exit 2 + ;; +esac From cac764f06b58612f6d781504729c27478f3f7d60 Mon Sep 17 00:00:00 2001 From: Naseem Alnaji Date: Thu, 22 Oct 2026 20:28:45 +0000 Subject: [PATCH 2/2] ci: run each package probe on the lowest PHP its mcp/sdk release allows The package probe ran on PHP 8.3 with every discovered mcp/sdk release. A release that needs PHP 8.4 cannot install there, so every pull request would fail without an AgentCat defect. ReleaseDiscovery now expands Packagist's minified rows, reads each release's PHP requirement with composer/semver, and gives each row the lowest PHP that CI tests and the release allows. - The nightly also runs the package probe on each discovered mcp/sdk release. - A nightly issue names the minor release, so a failure that lasts across its patches stays in one issue. The issue shows the end of the install, PHPStan and PHPUnit output, or the message of released-versions.php. - released-versions.php tries a failed GET twice more, and exits 1 for an unreadable composer.json as its header says. - The discover steps make sure that the output is a JSON list that is not empty before the matrix reads it. - require-library.sh requires the version without an update, then runs a full update, so every root package can move. - composer/semver is a dev dependency now; it came only through orchestra. --- .github/workflows/ci.yml | 16 +++-- .github/workflows/nightly.yml | 76 ++++++++++++++++++---- CONTRIBUTING.md | 2 +- composer.json | 1 + phpstan.no-bundle.neon | 2 +- tests/Architecture/WorkflowsTest.php | 47 ++++++++++++-- tests/Support/ReleaseDiscovery.php | 97 ++++++++++++++++++++++++---- tests/Unit/ReleaseDiscoveryTest.php | 80 ++++++++++++++++++++--- tests/bin/released-versions.php | 47 +++++++++----- tests/bin/require-library.sh | 9 ++- 10 files changed, 310 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec081a9..ad9c02b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,10 +181,11 @@ jobs: - name: Read the releases from Packagist id: releases run: | - releases="$(php tests/bin/released-versions.php)" + releases="$(php -d display_errors=stderr tests/bin/released-versions.php)" + jq -e 'type == "array" and length > 0' <<<"$releases" >/dev/null echo "releases=${releases}" >>"$GITHUB_OUTPUT" - echo "mcp-sdk=$(jq -c 'map(select(.library == "mcp/sdk") | .version)' <<<"$releases")" >>"$GITHUB_OUTPUT" - jq -r '.[] | "- \(.library) \(.version)"' <<<"$releases" >>"$GITHUB_STEP_SUMMARY" + echo "mcp-sdk=$(jq -c 'map(select(.library == "mcp/sdk"))' <<<"$releases")" >>"$GITHUB_OUTPUT" + jq -r '.[] | "- \(.library) \(.version) (lowest PHP \(.php))"' <<<"$releases" >>"$GITHUB_STEP_SUMMARY" released: name: ${{ matrix.library }} ${{ matrix.version }} (PHP 8.5) @@ -239,26 +240,27 @@ jobs: AGENTCAT_RUNTIME_TESTS: '1' package: - name: Package probe (PHP 8.3, mcp/sdk ${{ matrix.mcp-sdk }}) + name: Package probe (PHP ${{ matrix.php }}, mcp/sdk ${{ matrix.version }}) needs: discover runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: - mcp-sdk: ${{ fromJSON(needs.discover.outputs.mcp-sdk) }} + # Each mcp/sdk release, on the lowest PHP that CI tests and the release allows (8.3 today). + include: ${{ fromJSON(needs.discover.outputs.mcp-sdk) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # 2.40.0 with: - php-version: '8.3' + php-version: ${{ matrix.php }} coverage: none - name: Install the dist in a fresh project and track an mcp/sdk server run: tests/bin/package-probe.sh "$MCP_SDK" env: - MCP_SDK: ${{ matrix.mcp-sdk }} + MCP_SDK: ${{ matrix.version }} bc: name: Backward compatibility diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 75e627f..925abbc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -156,6 +156,7 @@ jobs: timeout-minutes: 5 outputs: releases: ${{ steps.releases.outputs.releases }} + mcp-sdk: ${{ steps.releases.outputs.mcp-sdk }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -171,10 +172,14 @@ jobs: - name: Read the releases from Packagist id: releases timeout-minutes: 2 + env: + ERRORS: ${{ runner.temp }}/discover.txt run: | - releases="$(php tests/bin/released-versions.php)" + releases="$(php -d display_errors=stderr tests/bin/released-versions.php 2> >(tee "$ERRORS" >&2))" + jq -e 'type == "array" and length > 0' <<<"$releases" >/dev/null echo "releases=${releases}" >>"$GITHUB_OUTPUT" - jq -r '.[] | "- \(.library) \(.version)"' <<<"$releases" >>"$GITHUB_STEP_SUMMARY" + echo "mcp-sdk=$(jq -c 'map(select(.library == "mcp/sdk"))' <<<"$releases")" >>"$GITHUB_OUTPUT" + jq -r '.[] | "- \(.library) \(.version) (lowest PHP \(.php))"' <<<"$releases" >>"$GITHUB_STEP_SUMMARY" - name: Fail on purpose (workflow_dispatch force_failure) if: github.event_name == 'workflow_dispatch' && inputs.force_failure == github.job run: exit 1 @@ -182,7 +187,12 @@ jobs: if: failure() && github.event_name != 'pull_request' env: GH_TOKEN: ${{ github.token }} - run: .github/nightly-issue.sh "${{ github.job }}" + ERRORS: ${{ runner.temp }}/discover.txt + REPORT: ${{ runner.temp }}/discover.md + run: | + # The message of released-versions.php, when it wrote one: the issue says why the releases could not be read. + if [ -s "$ERRORS" ]; then { printf '~~~text\n'; cat "$ERRORS"; printf '~~~\n'; } >"$REPORT"; fi + .github/nightly-issue.sh "${{ github.job }}" "$REPORT" released: name: ${{ matrix.library }} ${{ matrix.version }} (PHP 8.5) @@ -204,21 +214,30 @@ jobs: - uses: ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda # 4.0.0 with: dependency-versions: highest + # Each step adds its output to $LOG, so the issue shows why the install, PHPStan or the tests failed. - name: Install ${{ matrix.library }} ${{ matrix.version }} timeout-minutes: 8 env: LIBRARY: ${{ matrix.library }} VERSION: ${{ matrix.version }} - run: tests/bin/require-library.sh "$LIBRARY" "$VERSION" - - run: vendor/bin/phpstan analyse --no-progress --error-format=github --memory-limit=1G --configuration=${{ matrix.library == 'mcp/sdk' && 'phpstan.no-bundle.neon' || 'phpstan.neon.dist' }} + LOG: ${{ runner.temp }}/released.txt + run: | + set -o pipefail + tests/bin/require-library.sh "$LIBRARY" "$VERSION" 2>&1 | tee -a "$LOG" + - name: PHPStan timeout-minutes: 6 + env: + LOG: ${{ runner.temp }}/released.txt + run: | + set -o pipefail + vendor/bin/phpstan analyse --no-progress --error-format=github --memory-limit=1G --configuration=${{ matrix.library == 'mcp/sdk' && 'phpstan.no-bundle.neon' || 'phpstan.neon.dist' }} 2>&1 | tee -a "$LOG" - name: Run the tests timeout-minutes: 12 env: - OUTPUT: ${{ runner.temp }}/phpunit.txt + LOG: ${{ runner.temp }}/released.txt run: | set -o pipefail - vendor/bin/phpunit --testsuite Contract,Integration,E2E --no-progress --colors=never 2>&1 | tee "$OUTPUT" + vendor/bin/phpunit --testsuite Contract,Integration,E2E --no-progress --colors=never 2>&1 | tee -a "$LOG" - name: Fail on purpose (workflow_dispatch force_failure) if: github.event_name == 'workflow_dispatch' && inputs.force_failure == github.job run: exit 1 @@ -228,12 +247,45 @@ jobs: GH_TOKEN: ${{ github.token }} LIBRARY: ${{ matrix.library }} VERSION: ${{ matrix.version }} - OUTPUT: ${{ runner.temp }}/phpunit.txt - REPORT: ${{ runner.temp }}/phpunit.md + LOG: ${{ runner.temp }}/released.txt + REPORT: ${{ runner.temp }}/released.md run: | - # The failing tests and their messages, STDERR included, when PHPUnit ran: the issue names the failing test. - if [ -s "$OUTPUT" ]; then { printf '~~~text\n'; cat "$OUTPUT"; printf '~~~\n'; } >"$REPORT"; fi - .github/nightly-issue.sh "released (${LIBRARY} ${VERSION})" "$REPORT" + # The end of the log: the failing step writes last, and an issue body holds at most 65536 characters. + if [ -s "$LOG" ]; then { printf '~~~text\n'; tail -c 50000 "$LOG"; printf '\n~~~\n'; } >"$REPORT"; fi + # One issue per minor release: a failure that lasts across its patches stays in one issue. + .github/nightly-issue.sh "released (${LIBRARY} $(cut -d. -f1-2 <<<"$VERSION"))" "$REPORT" + + package: + name: Package probe (PHP ${{ matrix.php }}, mcp/sdk ${{ matrix.version }}) + needs: discover + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.discover.outputs.mcp-sdk) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # 2.40.0 + with: + php-version: ${{ matrix.php }} + coverage: none + - name: Install the dist in a fresh project and track an mcp/sdk server + timeout-minutes: 8 + env: + MCP_SDK: ${{ matrix.version }} + run: tests/bin/package-probe.sh "$MCP_SDK" + - name: Fail on purpose (workflow_dispatch force_failure) + if: github.event_name == 'workflow_dispatch' && inputs.force_failure == github.job + run: exit 1 + - name: Open or update the nightly issue + if: failure() && github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ matrix.version }} + run: .github/nightly-issue.sh "package (mcp/sdk $(cut -d. -f1-2 <<<"$VERSION"))" php-nightly: name: Tests (PHP nightly) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c00056d..ea7b570 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,7 +90,7 @@ git commit -m "docs: update API documentation" ### The Required Check -The one required check is `All green`. It passes when every CI job passes: static analysis on the lowest and the highest dependencies, the tests on PHP 8.3, 8.4 and 8.5, coverage, mutation testing, the released-version legs, the runtime suite, the package probe and the backward-compatibility check. The released-version legs and the package probe take their versions from Packagist on each run: the newest patch of each minor release of `mcp/sdk`, `laravel/mcp` and `symfony/mcp-bundle`, from the lowest version that `require-dev` allows. The nightly runs the same legs, so a release published between pull requests opens an issue. +The one required check is `All green`. It passes when every CI job passes: static analysis on the lowest and the highest dependencies, the tests on PHP 8.3, 8.4 and 8.5, coverage, mutation testing, the released-version legs, the runtime suite, the package probe and the backward-compatibility check. The released-version legs take their versions from Packagist on each run: the newest patch of each minor release of `mcp/sdk`, `laravel/mcp` and `symfony/mcp-bundle`, from the lowest version that `require-dev` allows. The package probe runs on each of those `mcp/sdk` releases. The nightly runs the legs and the probe again, so a release published between pull requests that breaks them opens an issue. When the install step of a leg fails, `require-dev` does not allow that release yet: widen its range, or the range of the framework that it needs. ### No Issue Required diff --git a/composer.json b/composer.json index 97338f2..43b057f 100644 --- a/composer.json +++ b/composer.json @@ -54,6 +54,7 @@ }, "require-dev": { "brianium/paratest": "^7.8.6", + "composer/semver": "^3.4", "ergebnis/composer-normalize": "^2.54", "ergebnis/phpunit-slow-test-detector": "^2.26", "infection/infection": "^0.35.8", diff --git a/phpstan.no-bundle.neon b/phpstan.no-bundle.neon index fda9eab..7879812 100644 --- a/phpstan.no-bundle.neon +++ b/phpstan.no-bundle.neon @@ -1,4 +1,4 @@ -# PHPStan for the mcp/sdk main leg, which removes symfony/mcp-bundle (it requires mcp/sdk ^0.8.1). +# PHPStan for the mcp/sdk legs (released versions and main), which remove symfony/mcp-bundle (it requires mcp/sdk ^0.8.1). includes: - phpstan.neon.dist parameters: diff --git a/tests/Architecture/WorkflowsTest.php b/tests/Architecture/WorkflowsTest.php index f197f44..0d2b43a 100644 --- a/tests/Architecture/WorkflowsTest.php +++ b/tests/Architecture/WorkflowsTest.php @@ -4,14 +4,15 @@ namespace AgentCat\Tests\Architecture; +use AgentCat\Tests\Support\ReleaseDiscovery; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Symfony\Component\Yaml\Yaml; /** * The jobs of the workflows that the required check and the nightly issue depend on. `All green` needs every CI job - * that gates, and the `released` jobs of CI and the nightly test the releases that the `discover` job reads from - * Packagist (D72), so a new release joins the matrix without a commit. + * that gates, and the `released` and `package` jobs of CI and the nightly test the releases that the `discover` job + * reads from Packagist (D72), so a new release joins the matrix without a commit. */ final class WorkflowsTest extends TestCase { @@ -19,6 +20,8 @@ final class WorkflowsTest extends TestCase private const string ISSUE_STEP = 'Open or update the nightly issue'; + private const string SETUP_PHP = 'shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962'; + public function testAllGreenNeedsEveryCiJobThatGates(): void { $jobs = $this->jobs('ci.yml'); @@ -49,7 +52,8 @@ public function testTheReleasedJobsTestTheDiscoveredReleases(string $workflow): { $jobs = $this->jobs($workflow); - $this->assertStringContainsString('php tests/bin/released-versions.php', $this->runs($jobs['discover'] ?? [])); + $this->assertStringContainsString('tests/bin/released-versions.php', $this->runs($jobs['discover'] ?? [])); + $this->assertStringContainsString('echo "releases=${releases}" >>"$GITHUB_OUTPUT"', $this->stepRun($jobs, 'discover', 'releases')); $this->assertSame('${{ steps.releases.outputs.releases }}', $this->at($jobs, 'discover', 'outputs', 'releases')); $this->assertSame('discover', $this->at($jobs, 'released', 'needs')); $this->assertSame('${{ fromJSON(needs.discover.outputs.releases) }}', $this->at($jobs, 'released', 'strategy', 'matrix', 'include')); @@ -58,12 +62,26 @@ public function testTheReleasedJobsTestTheDiscoveredReleases(string $workflow): $this->assertStringContainsString('vendor/bin/phpunit --testsuite Contract,Integration,E2E', $this->runs($jobs['released'] ?? [])); } - public function testThePackageProbeRunsOnEachDiscoveredMcpSdkRelease(): void + /** Each mcp/sdk release runs the probe on the lowest PHP that CI tests and the release allows. */ + #[DataProvider('workflows')] + public function testThePackageProbeRunsOnEachDiscoveredMcpSdkRelease(string $workflow): void { - $jobs = $this->jobs('ci.yml'); + $jobs = $this->jobs($workflow); + $this->assertStringContainsString('echo "mcp-sdk=$(jq -c \'map(select(.library == "mcp/sdk"))\'', $this->stepRun($jobs, 'discover', 'releases')); + $this->assertSame('${{ steps.releases.outputs.mcp-sdk }}', $this->at($jobs, 'discover', 'outputs', 'mcp-sdk')); $this->assertSame('discover', $this->at($jobs, 'package', 'needs')); - $this->assertSame('${{ fromJSON(needs.discover.outputs.mcp-sdk) }}', $this->at($jobs, 'package', 'strategy', 'matrix', 'mcp-sdk')); + $this->assertSame('${{ fromJSON(needs.discover.outputs.mcp-sdk) }}', $this->at($jobs, 'package', 'strategy', 'matrix', 'include')); + $steps = $this->at($jobs, 'package', 'steps'); + $this->assertIsArray($steps); + $this->assertContains(['uses' => self::SETUP_PHP, 'with' => ['php-version' => '${{ matrix.php }}', 'coverage' => 'none']], $steps); + $this->assertStringContainsString('tests/bin/package-probe.sh "$MCP_SDK"', $this->runs($jobs['package'] ?? [])); + } + + /** The discovery picks each probe's PHP from the versions that the tests job runs. */ + public function testTheDiscoveryKnowsThePhpVersionsThatCiTests(): void + { + $this->assertSame(ReleaseDiscovery::PHP_VERSIONS, $this->at($this->jobs('ci.yml'), 'tests', 'strategy', 'matrix', 'php')); } /** A scheduled failure reaches a person: every nightly job ends by opening or updating its issue. */ @@ -110,6 +128,23 @@ private function at(array $value, string ...$keys): mixed return $at; } + /** + * The `run` script of the step with this `id`. + * + * @param array $jobs + */ + private function stepRun(array $jobs, string $job, string $id): string + { + $steps = $this->at($jobs, $job, 'steps'); + $this->assertIsArray($steps); + foreach ($steps as $step) { + if (is_array($step) && ($step['id'] ?? null) === $id && is_string($step['run'] ?? null)) { + return $step['run']; + } + } + self::fail(sprintf('the %s job has no step with id %s', $job, $id)); + } + /** * The `run` scripts of a job's steps, one after the other. * diff --git a/tests/Support/ReleaseDiscovery.php b/tests/Support/ReleaseDiscovery.php index 3965603..0973a00 100644 --- a/tests/Support/ReleaseDiscovery.php +++ b/tests/Support/ReleaseDiscovery.php @@ -4,23 +4,34 @@ namespace AgentCat\Tests\Support; +use Composer\Semver\VersionParser; + /** - * The releases of the MCP libraries that the `released` jobs of ci.yml and nightly.yml test (D72): the newest patch of - * each minor version, from the lowest version that `require-dev` allows upward. Like the TypeScript SDK's - * `mcp-compatibility.yml`, `tests/bin/released-versions.php` reads them from Packagist on each run, so a new release - * joins the matrix without a commit. + * The releases of the MCP libraries that the `released` and `package` jobs of ci.yml and nightly.yml test (D72): the + * newest patch of each minor version, from the lowest version that `require-dev` allows upward. Like the TypeScript + * SDK's `mcp-compatibility.yml`, `tests/bin/released-versions.php` reads them from Packagist on each run, so a new + * release joins the matrix without a commit. */ final class ReleaseDiscovery { /** The libraries that the adapters integrate, in the order of the matrix. */ public const array LIBRARIES = ['mcp/sdk', 'laravel/mcp', 'symfony/mcp-bundle']; + /** The PHP versions that CI tests, lowest first: the `php` matrix of the `tests` job in ci.yml. */ + public const array PHP_VERSIONS = ['8.3', '8.4', '8.5']; + /** A `^` constraint: its base is the lowest version it allows. */ private const string CARET = '/\A\^(\d+)\.(\d+)(?:\.(\d+))?\z/'; - /** A stable release in Packagist's `version_normalized` form: four numbers, no stability suffix. */ + /** + * A stable release in Packagist's `version_normalized` form: four numbers, no stability suffix. A `-patchN` release, + * which Composer also calls stable, does not count: none of the three libraries publishes one. + */ private const string STABLE = '/\A(\d+)\.(\d+)\.(\d+)\.(\d+)\z/'; + /** The value of a key that a minified Packagist row removes. */ + private const string UNSET = '__unset'; + /** * The floor of each library: the lowest version of its `require-dev` constraint, which the lowest-dependencies jobs * test. @@ -68,12 +79,42 @@ public static function metadataUrl(string $library): string } /** - * The stable versions of a Packagist `p2` body, in its order, as major.minor.patch (a fourth number only when it is - * not 0). Packagist minifies the body, but each row keeps its own `version_normalized`. + * The rows of the matrix: for each library, the newest patch of each minor version at or above its floor, with the + * lowest PHP version of PHP_VERSIONS that the release allows. * - * @return list + * @param array $floors library => version + * @param \Closure(string): string $fetch the body at a URL + * + * @return list + * + * @throws \RangeException when a library has no release at or above its floor + * @throws \RuntimeException when `$fetch` fails or a body cannot be read + */ + public static function matrix(array $floors, \Closure $fetch): array + { + $rows = []; + foreach ($floors as $library => $floor) { + $releases = self::stableReleases($fetch(self::metadataUrl($library)), $library); + $versions = self::newestPatches(array_keys($releases), $floor); + if ($versions === []) { + throw new \RangeException(sprintf('Packagist has no release of %s at or above %s', $library, $floor)); + } + foreach ($versions as $version) { + $rows[] = ['library' => $library, 'version' => $version, 'php' => self::lowestPhp($releases[$version] ?? '*')]; + } + } + + return $rows; + } + + /** + * The stable releases of a Packagist `p2` body, in its order: the version as major.minor.patch (a fourth number only + * when it is not 0) => the release's PHP constraint, `*` when it sets none. Packagist minifies the body: a row holds + * the keys that changed since the row before it, and `__unset` removes a key, so the rows expand in order. + * + * @return array */ - public static function stableVersions(string $body, string $library): array + public static function stableReleases(string $body, string $library): array { $metadata = json_decode($body, true); if (!is_array($metadata)) { @@ -83,18 +124,48 @@ public static function stableVersions(string $body, string $library): array if (!is_array($rows)) { throw new \UnexpectedValueException('the Packagist body has no versions of ' . $library); } - $versions = []; + $releases = []; + $expanded = []; foreach ($rows as $row) { - $normalized = is_array($row) ? $row['version_normalized'] ?? null : null; + if (!is_array($row)) { + throw new \UnexpectedValueException('a version of ' . $library . ' on Packagist is not an object'); + } + foreach ($row as $key => $value) { + if ($value === self::UNSET) { + unset($expanded[$key]); + } else { + $expanded[$key] = $value; + } + } + $normalized = $expanded['version_normalized'] ?? null; if (!is_string($normalized)) { throw new \UnexpectedValueException('a version of ' . $library . ' on Packagist has no version_normalized'); } if (preg_match(self::STABLE, $normalized, $parts) === 1) { - $versions[] = $parts[1] . '.' . $parts[2] . '.' . $parts[3] . ($parts[4] === '0' ? '' : '.' . $parts[4]); + $version = $parts[1] . '.' . $parts[2] . '.' . $parts[3] . ($parts[4] === '0' ? '' : '.' . $parts[4]); + $php = is_array($expanded['require'] ?? null) ? $expanded['require']['php'] ?? null : null; + $releases[$version] = is_string($php) ? $php : '*'; + } + } + + return $releases; + } + + /** + * The lowest version of PHP_VERSIONS that a PHP constraint allows. When it allows none, the newest: Composer then + * stops the install with the release's PHP requirement, which says that CI needs a newer PHP. + */ + public static function lowestPhp(string $constraint): string + { + $parser = new VersionParser(); + $allowed = $parser->parseConstraints($constraint); + foreach (self::PHP_VERSIONS as $php) { + if ($allowed->matches($parser->parseConstraints($php . '.*'))) { + return $php; } } - return $versions; + return self::PHP_VERSIONS[count(self::PHP_VERSIONS) - 1]; } /** diff --git a/tests/Unit/ReleaseDiscoveryTest.php b/tests/Unit/ReleaseDiscoveryTest.php index 383a401..4af8aff 100644 --- a/tests/Unit/ReleaseDiscoveryTest.php +++ b/tests/Unit/ReleaseDiscoveryTest.php @@ -90,23 +90,25 @@ public function testTheRepositoryFloorsMatchItsConflictFloors(): void $expected = array_intersect_key($floors, $conflictFloors); ksort($expected); - $this->assertSame(['mcp/sdk', 'laravel/mcp', 'symfony/mcp-bundle'], array_keys($floors)); - $this->assertSame(['laravel/mcp', 'mcp/sdk'], array_keys($conflictFloors), 'symfony/mcp-bundle has no conflict floor'); + $this->assertSame(ReleaseDiscovery::LIBRARIES, array_keys($floors)); + $this->assertArrayHasKey('mcp/sdk', $conflictFloors); + $this->assertArrayHasKey('laravel/mcp', $conflictFloors); $this->assertSame($expected, $conflictFloors); } - public function testTheStableVersionsOfAPackagistBody(): void + /** Packagist minifies the rows: a row inherits each key it leaves out, and `__unset` removes one. */ + public function testTheStableReleasesOfAMinifiedPackagistBody(): void { $body = $this->packagist('mcp/sdk', [ - ['version' => 'v0.9.0', 'version_normalized' => '0.9.0.0', 'require' => ['php' => '^8.1']], - ['version' => 'v0.9.0-RC1', 'version_normalized' => '0.9.0.0-RC1'], + ['version' => 'v0.9.0', 'version_normalized' => '0.9.0.0', 'require' => ['php' => '>=8.4']], + ['version' => 'v0.9.0-RC1', 'version_normalized' => '0.9.0.0-RC1', 'require' => ['php' => '^8.1']], ['version' => 'v0.8.1', 'version_normalized' => '0.8.1.0'], ['version' => '0.8.0-beta.2', 'version_normalized' => '0.8.0.0-beta2'], ['version' => 'dev-main', 'version_normalized' => 'dev-main'], - ['version' => '1.2.3.4', 'version_normalized' => '1.2.3.4'], + ['version' => '1.2.3.4', 'version_normalized' => '1.2.3.4', 'require' => '__unset'], ]); - $this->assertSame(['0.9.0', '0.8.1', '1.2.3.4'], ReleaseDiscovery::stableVersions($body, 'mcp/sdk')); + $this->assertSame(['0.9.0' => '>=8.4', '0.8.1' => '^8.1', '1.2.3.4' => '*'], ReleaseDiscovery::stableReleases($body, 'mcp/sdk')); } /** @return iterable */ @@ -116,16 +118,78 @@ public static function unreadableBodies(): iterable yield 'another package' => ['{"packages":{"laravel/mcp":[]}}', 'the Packagist body has no versions of mcp/sdk']; yield 'no packages' => ['{"minified":"composer/2.0"}', 'the Packagist body has no versions of mcp/sdk']; yield 'a row without its version' => ['{"packages":{"mcp/sdk":[{"version":"v0.9.0"}]}}', 'a version of mcp/sdk on Packagist has no version_normalized']; + yield 'a row that is no object' => ['{"packages":{"mcp/sdk":["0.9.0"]}}', 'a version of mcp/sdk on Packagist is not an object']; } #[DataProvider('unreadableBodies')] public function testABodyItCannotReadStopsTheRun(string $body, string $message): void { - $thrown = Thrown::by(\UnexpectedValueException::class, static fn(): array => ReleaseDiscovery::stableVersions($body, 'mcp/sdk')); + $thrown = Thrown::by(\UnexpectedValueException::class, static fn(): array => ReleaseDiscovery::stableReleases($body, 'mcp/sdk')); $this->assertSame($message, $thrown->getMessage()); } + /** @return iterable */ + public static function phpConstraints(): iterable + { + yield 'every version' => ['^8.1', '8.3']; + yield 'no requirement' => ['*', '8.3']; + yield 'a newer floor' => ['>=8.4', '8.4']; + yield 'two carets' => ['^8.2 || ^8.3', '8.3']; + yield 'a patch floor' => ['^8.4.2', '8.4']; + yield 'one minor' => ['~8.5.0', '8.5']; + yield 'a version CI does not test' => ['^8.6', '8.5']; + } + + /** + * The package probe runs each release on the lowest PHP that CI tests and the release allows, so a release that + * drops PHP 8.3 still runs. A release that allows none runs on the newest, where Composer names its requirement. + */ + #[DataProvider('phpConstraints')] + public function testTheLowestPhpVersionThatTheReleaseAllows(string $constraint, string $php): void + { + $this->assertSame($php, ReleaseDiscovery::lowestPhp($constraint)); + } + + public function testTheMatrixHoldsTheNewestPatchOfEachMinorOfEachLibrary(): void + { + $bodies = [ + 'https://repo.packagist.org/p2/mcp/sdk.json' => $this->packagist('mcp/sdk', [ + ['version_normalized' => '0.10.0.0', 'require' => ['php' => '>=8.4']], + ['version_normalized' => '0.9.1.0', 'require' => ['php' => '^8.1']], + ['version_normalized' => '0.9.0.0'], + ['version_normalized' => '0.8.1.0'], + ]), + 'https://repo.packagist.org/p2/laravel/mcp.json' => $this->packagist('laravel/mcp', [ + ['version_normalized' => '1.0.0.0', 'require' => ['php' => '^8.2']], + ]), + ]; + $fetched = []; + $fetch = static function (string $url) use ($bodies, &$fetched): string { + $fetched[] = $url; + + return $bodies[$url] ?? throw new \RuntimeException('GET ' . $url . ': HTTP 404'); + }; + + $rows = ReleaseDiscovery::matrix(['mcp/sdk' => '0.9.0', 'laravel/mcp' => '1.0.0'], $fetch); + + $this->assertSame([ + ['library' => 'mcp/sdk', 'version' => '0.9.1', 'php' => '8.3'], + ['library' => 'mcp/sdk', 'version' => '0.10.0', 'php' => '8.4'], + ['library' => 'laravel/mcp', 'version' => '1.0.0', 'php' => '8.3'], + ], $rows); + $this->assertSame(array_keys($bodies), $fetched); + } + + public function testALibraryWithoutAReleaseAtItsFloorStopsTheRun(): void + { + $body = $this->packagist('mcp/sdk', [['version_normalized' => '0.7.1.0']]); + + $thrown = Thrown::by(\RangeException::class, static fn(): array => ReleaseDiscovery::matrix(['mcp/sdk' => '0.8.1'], static fn(): string => $body)); + + $this->assertSame('Packagist has no release of mcp/sdk at or above 0.8.1', $thrown->getMessage()); + } + /** @return iterable, string, list}> */ public static function releases(): iterable { diff --git a/tests/bin/released-versions.php b/tests/bin/released-versions.php index 4c6b748..cce35c7 100644 --- a/tests/bin/released-versions.php +++ b/tests/bin/released-versions.php @@ -4,31 +4,46 @@ // Usage: php tests/bin/released-versions.php // -// Prints the releases that the `released` jobs of ci.yml and nightly.yml test (D72), as one line of JSON: a list of -// {"library": ..., "version": ...}. For mcp/sdk, laravel/mcp and symfony/mcp-bundle, they are the newest patch of each -// minor version from the lowest version that composer.json's require-dev allows, read from Packagist on each run. -// Exits 1 when a library has no release at or above that floor, or 2 when Packagist cannot be read. +// Prints the releases that the `released` and `package` jobs of ci.yml and nightly.yml test (D72), as one line of +// JSON: a list of {"library", "version", "php"}. For mcp/sdk, laravel/mcp and symfony/mcp-bundle, they are the newest +// patch of each minor version from the lowest version that composer.json's require-dev allows, read from Packagist on +// each run; "php" is the lowest PHP version that CI tests and the release allows. Exits 1 when composer.json has no +// floor it can read or a library has no release at or above its floor, or 2 when Packagist cannot be read. use AgentCat\Tests\Support\HttpGet; use AgentCat\Tests\Support\ReleaseDiscovery; require __DIR__ . '/../../vendor/autoload.php'; -$floors = ReleaseDiscovery::floors((string) file_get_contents(__DIR__ . '/../../composer.json')); - -$rows = []; try { - foreach ($floors as $library => $floor) { - $body = HttpGet::body(ReleaseDiscovery::metadataUrl($library), 'released-versions'); - $versions = ReleaseDiscovery::newestPatches(ReleaseDiscovery::stableVersions($body, $library), $floor); - if ($versions === []) { - fwrite(STDERR, sprintf("released-versions: Packagist has no release of %s at or above %s\n", $library, $floor)); - exit(1); - } - foreach ($versions as $version) { - $rows[] = ['library' => $library, 'version' => $version]; + $floors = ReleaseDiscovery::floors((string) file_get_contents(__DIR__ . '/../../composer.json')); +} catch (Throwable $e) { + fwrite(STDERR, 'released-versions: composer.json: ' . $e->getMessage() . "\n"); + exit(1); +} + +// One failed GET is tried twice more, after 2 s and then 5 s: Packagist often answers again after a short pause. +$fetch = static function (string $url): string { + $pauses = [2, 5]; + while (true) { + try { + return HttpGet::body($url, 'released-versions'); + } catch (RuntimeException $e) { + $pause = array_shift($pauses); + if ($pause === null) { + throw $e; + } + fwrite(STDERR, sprintf("released-versions: %s; trying again in %d s\n", $e->getMessage(), $pause)); + sleep($pause); } } +}; + +try { + $rows = ReleaseDiscovery::matrix($floors, $fetch); +} catch (RangeException $e) { + fwrite(STDERR, 'released-versions: ' . $e->getMessage() . "\n"); + exit(1); } catch (RuntimeException $e) { fwrite(STDERR, 'released-versions: ' . $e->getMessage() . "\n"); exit(2); diff --git a/tests/bin/require-library.sh b/tests/bin/require-library.sh index 97c3b50..733d737 100755 --- a/tests/bin/require-library.sh +++ b/tests/bin/require-library.sh @@ -11,17 +11,20 @@ case "$library" in # symfony/mcp-bundle requires mcp/sdk ^0.8.1, which 0.9 and main do not satisfy: the Symfony tests skip without # it, and the bundle's own rows test it. composer remove --dev --no-update symfony/mcp-bundle - composer require --dev --no-interaction --update-with-all-dependencies "mcp/sdk:${constraint}" + composer require --dev --no-update "mcp/sdk:${constraint}" ;; laravel/mcp) - composer require --dev --no-interaction --update-with-all-dependencies "laravel/mcp:${constraint}" + composer require --dev --no-update "laravel/mcp:${constraint}" ;; symfony/mcp-bundle) # The bundle chooses its mcp/sdk, so a release that needs a newer mcp/sdk than require-dev allows still installs. - composer require --dev --no-interaction --update-with-all-dependencies "symfony/mcp-bundle:${constraint}" "mcp/sdk:*" + composer require --dev --no-update "symfony/mcp-bundle:${constraint}" "mcp/sdk:*" ;; *) echo "require-library.sh: unknown library ${library}" >&2 exit 2 ;; esac +# Every root package can move, so a release that needs a newer shared dependency still installs. When this update +# fails, require-dev does not allow that release yet (for example a new Laravel or Symfony major). +composer update --no-interaction