diff --git a/CHANGELOG.md b/CHANGELOG.md index 64cc937..d95c993 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to the rt_llms_txt extension will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- New site setting `llmsTxt.enableMarkdown` (default: enabled) to turn off the `.md` Markdown variant per site. When disabled, `.md` URLs are left untouched by `UrlSuffixMiddleware` (so they 404 normally instead of being rewritten), `ContentFormatMiddleware` refuses to render Markdown as a safety net, and `llms.txt` no longer advertises the Markdown format or lists per-page Markdown links. + ## [1.0.12] - 2026-04-25 ### Fixed diff --git a/Classes/Middleware/ContentFormatMiddleware.php b/Classes/Middleware/ContentFormatMiddleware.php index ea1e3d9..05dd4f0 100644 --- a/Classes/Middleware/ContentFormatMiddleware.php +++ b/Classes/Middleware/ContentFormatMiddleware.php @@ -16,6 +16,7 @@ use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction; use TYPO3\CMS\Core\Http\Response; +use TYPO3\CMS\Core\Site\Entity\Site; use TYPO3\CMS\Core\Site\Entity\SiteLanguage; /** @@ -49,6 +50,14 @@ public function process(ServerRequestInterface $request, RequestHandlerInterface return $handler->handle($request); } + // Safety net: never render Markdown for a site that has it disabled, + // regardless of how a request reached this point (defense in depth + // alongside UrlSuffixMiddleware, which normally prevents this). + $site = $request->getAttribute('site'); + if ($site instanceof Site && !(bool)$site->getSettings()->get('llmsTxt.enableMarkdown', true)) { + return $handler->handle($request); + } + // Check API key protection (via ApiKeyAuthenticationTrait) $authResponse = $this->checkApiKeyAuth($request); if ($authResponse instanceof ResponseInterface) { diff --git a/Classes/Middleware/UrlSuffixMiddleware.php b/Classes/Middleware/UrlSuffixMiddleware.php index 5241a7e..15627cd 100644 --- a/Classes/Middleware/UrlSuffixMiddleware.php +++ b/Classes/Middleware/UrlSuffixMiddleware.php @@ -8,6 +8,8 @@ use Psr\Http\Message\ServerRequestInterface; use Psr\Http\Server\MiddlewareInterface; use Psr\Http\Server\RequestHandlerInterface; +use TYPO3\CMS\Core\Routing\SiteMatcher; +use TYPO3\CMS\Core\Routing\SiteRouteResult; /** * Middleware that detects .md suffix in URLs and rewrites them for routing. @@ -17,25 +19,46 @@ * 2. Strip the suffix and set a request attribute for format detection * 3. Allow normal TYPO3 routing to find the actual page * + * If the target site has Markdown output disabled (llmsTxt.enableMarkdown), + * the .md suffix is left untouched so normal TYPO3 routing 404s it instead + * of exposing any Markdown/content-format handling further down the stack. + * * Spec-compliant with https://llmstxt.org/ */ -final class UrlSuffixMiddleware implements MiddlewareInterface +final readonly class UrlSuffixMiddleware implements MiddlewareInterface { public const REQUEST_ATTRIBUTE = 'llms_txt_format'; private const MARKDOWN_SUFFIX = '.md'; private const INDEX_SUFFIX = '/index.html.md'; + public function __construct( + private SiteMatcher $siteMatcher, + ) {} + public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface { $uri = $request->getUri(); $path = $uri->getPath(); - // Check for .md suffix - if (str_ends_with($path, self::INDEX_SUFFIX)) { + $isIndexSuffix = str_ends_with($path, self::INDEX_SUFFIX); + $isMarkdownSuffix = !$isIndexSuffix && str_ends_with($path, self::MARKDOWN_SUFFIX); + + if (!$isIndexSuffix && !$isMarkdownSuffix) { + return $handler->handle($request); + } + + // Site isn't resolved yet at this point in the middleware stack (this + // middleware runs before the site resolver on purpose), so the site + // has to be matched independently to read its Markdown setting. + if (!$this->isMarkdownEnabledForRequest($request)) { + return $handler->handle($request); + } + + if ($isIndexSuffix) { // /page/index.html.md -> /page/ $newPath = substr($path, 0, -\strlen(self::INDEX_SUFFIX) + 1); $request = $this->rewriteRequest($request, $newPath); - } elseif (str_ends_with($path, self::MARKDOWN_SUFFIX)) { + } else { // /page.md -> /page or /page/.md -> /page/ $newPath = substr($path, 0, -\strlen(self::MARKDOWN_SUFFIX)); // Handle /page/.md edge case @@ -52,6 +75,22 @@ public function process(ServerRequestInterface $request, RequestHandlerInterface return $handler->handle($request); } + /** + * Whether Markdown output is enabled for the site matching this request. + * + * Defaults to true (enabled) if no site can be matched yet, matching the + * setting's own default and preserving prior behaviour for edge cases. + */ + private function isMarkdownEnabledForRequest(ServerRequestInterface $request): bool + { + $routeResult = $this->siteMatcher->matchRequest($request); + if (!$routeResult instanceof SiteRouteResult) { + return true; + } + + return (bool)$routeResult->getSite()->getSettings()->get('llmsTxt.enableMarkdown', true); + } + /** * Rewrite the request with new path and set format attribute. */ diff --git a/Classes/Service/LlmsTxtGeneratorService.php b/Classes/Service/LlmsTxtGeneratorService.php index faf4c87..952f6c4 100644 --- a/Classes/Service/LlmsTxtGeneratorService.php +++ b/Classes/Service/LlmsTxtGeneratorService.php @@ -38,6 +38,7 @@ public function getContentForSite(Site $site): string $excludePages = $this->parseExcludePages($settings['excludePages'] ?? ''); $includeHidden = (bool)($settings['includeHidden'] ?? false); $intro = trim((string)($settings['intro'] ?? '')); + $enableMarkdown = (bool)($settings['enableMarkdown'] ?? true); $pages = $this->pageTreeService->getPages($site, $defaultLanguage, $excludePages, $includeHidden); @@ -58,7 +59,7 @@ public function getContentForSite(Site $site): string $apiKey = trim((string)($settings['apiKey'] ?? '')); - return $this->buildContent($site, $defaultLanguage, $pages, $baseUrl, $intro, $apiKey); + return $this->buildContent($site, $defaultLanguage, $pages, $baseUrl, $intro, $apiKey, $enableMarkdown); } /** @@ -90,6 +91,7 @@ private function buildContent( string $baseUrl, string $intro, string $apiKey, + bool $enableMarkdown, ): string { $lines = []; @@ -115,18 +117,21 @@ private function buildContent( $lines[] = '**Generated:** ' . date('Y-m-d H:i:s'); $lines[] = ''; - // Find an example page (first non-root page for realistic examples) - $examplePageUrl = $this->findExamplePageUrl($site, $sortedPages, $language); - // LLM-optimized content access section (spec-compliant with llmstxt.org) - $lines[] = '## LLM-Optimized Content Access'; - $lines[] = ''; - $lines[] = 'This site provides LLM-friendly Markdown output for all pages:'; - $lines[] = ''; - $lines[] = '### Markdown Format'; - $lines[] = 'Append `.md` to any page URL to get plain Markdown with YAML frontmatter.'; - $lines[] = '- **Example:** `' . $this->buildMarkdownUrl($examplePageUrl) . '`'; - $lines[] = ''; + // Omitted entirely when Markdown output is disabled for this site. + if ($enableMarkdown) { + // Find an example page (first non-root page for realistic examples) + $examplePageUrl = $this->findExamplePageUrl($site, $sortedPages, $language); + + $lines[] = '## LLM-Optimized Content Access'; + $lines[] = ''; + $lines[] = 'This site provides LLM-friendly Markdown output for all pages:'; + $lines[] = ''; + $lines[] = '### Markdown Format'; + $lines[] = 'Append `.md` to any page URL to get plain Markdown with YAML frontmatter.'; + $lines[] = '- **Example:** `' . $this->buildMarkdownUrl($examplePageUrl) . '`'; + $lines[] = ''; + } // Add authentication section if API key is configured if ($apiKey !== '') { @@ -140,7 +145,9 @@ private function buildContent( $lines[] = ''; $lines[] = '**Query Parameter:**'; $lines[] = '```'; - $lines[] = $baseUrl . '/page.md?api_key='; + $lines[] = $enableMarkdown + ? $baseUrl . '/page.md?api_key=' + : $baseUrl . '/llms.txt?api_key='; $lines[] = '```'; $lines[] = ''; } @@ -178,9 +185,11 @@ private function buildContent( $lines[] = str_repeat(' ', $indent) . ' > ' . str_replace("\n", ' ', $summary); } - // Add format access hints (spec-compliant .md suffix) - $mdUrl = $this->buildMarkdownUrl($pageUrl); - $lines[] = str_repeat(' ', $indent) . ' [Markdown](' . $mdUrl . ')'; + // Add format access hints (spec-compliant .md suffix), unless disabled + if ($enableMarkdown) { + $mdUrl = $this->buildMarkdownUrl($pageUrl); + $lines[] = str_repeat(' ', $indent) . ' [Markdown](' . $mdUrl . ')'; + } $lines[] = ''; } diff --git a/Configuration/Sets/LlmsTxt/settings.definitions.yaml b/Configuration/Sets/LlmsTxt/settings.definitions.yaml index 671964a..0356b15 100644 --- a/Configuration/Sets/LlmsTxt/settings.definitions.yaml +++ b/Configuration/Sets/LlmsTxt/settings.definitions.yaml @@ -27,6 +27,12 @@ settings: label: 'Include Hidden Pages' description: 'Include hidden pages in llms.txt generation' category: llmstxt + llmsTxt.enableMarkdown: + type: bool + default: true + label: 'Enable Markdown output' + description: 'Serve the .md Markdown variant for pages and reference it from llms.txt. If disabled, .md URLs are left untouched (404) and no Markdown links are listed in llms.txt.' + category: llmstxt llmsTxt.apiKey: type: string default: '' diff --git a/Documentation/Configuration.rst b/Documentation/Configuration.rst index 9164049..7224777 100644 --- a/Documentation/Configuration.rst +++ b/Documentation/Configuration.rst @@ -71,6 +71,18 @@ you can configure the extension in **Site Management > Settings**. If enabled, hidden pages are also included in the llms.txt generation. This can be useful for staging environments or preview purposes. +.. _confval-enableMarkdown: + +.. confval:: llmsTxt.enableMarkdown + + :type: boolean + :Default: true + + If disabled, the ``.md`` Markdown variant is no longer served for any + page on this site: ``.md`` URLs are left untouched and resolve as a + normal 404 instead of being rewritten, and llms.txt no longer lists a + "Markdown Format" section or per-page Markdown links. + .. _confval-apiKey: .. confval:: llmsTxt.apiKey