Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ All notable changes to the rt_llms_txt extension will be documented in this file
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]

### Added

- New site setting `llmsTxt.enableMarkdown` (default: enabled) to turn off the `.md` Markdown variant per site. When disabled, `.md` URLs are left untouched by `UrlSuffixMiddleware` (so they 404 normally instead of being rewritten), `ContentFormatMiddleware` refuses to render Markdown as a safety net, and `llms.txt` no longer advertises the Markdown format or lists per-page Markdown links.

## [1.0.12] - 2026-04-25

### Fixed
Expand Down
9 changes: 9 additions & 0 deletions Classes/Middleware/ContentFormatMiddleware.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
use TYPO3\CMS\Core\Http\Response;
use TYPO3\CMS\Core\Site\Entity\Site;
use TYPO3\CMS\Core\Site\Entity\SiteLanguage;

/**
Expand Down Expand Up @@ -49,6 +50,14 @@ public function process(ServerRequestInterface $request, RequestHandlerInterface
return $handler->handle($request);
}

// Safety net: never render Markdown for a site that has it disabled,
// regardless of how a request reached this point (defense in depth
// alongside UrlSuffixMiddleware, which normally prevents this).
$site = $request->getAttribute('site');
if ($site instanceof Site && !(bool)$site->getSettings()->get('llmsTxt.enableMarkdown', true)) {
return $handler->handle($request);
}

// Check API key protection (via ApiKeyAuthenticationTrait)
$authResponse = $this->checkApiKeyAuth($request);
if ($authResponse instanceof ResponseInterface) {
Expand Down
47 changes: 43 additions & 4 deletions Classes/Middleware/UrlSuffixMiddleware.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
use Psr\Http\Message\ServerRequestInterface;
use Psr\Http\Server\MiddlewareInterface;
use Psr\Http\Server\RequestHandlerInterface;
use TYPO3\CMS\Core\Routing\SiteMatcher;
use TYPO3\CMS\Core\Routing\SiteRouteResult;

/**
* Middleware that detects .md suffix in URLs and rewrites them for routing.
Expand All @@ -17,25 +19,46 @@
* 2. Strip the suffix and set a request attribute for format detection
* 3. Allow normal TYPO3 routing to find the actual page
*
* If the target site has Markdown output disabled (llmsTxt.enableMarkdown),
* the .md suffix is left untouched so normal TYPO3 routing 404s it instead
* of exposing any Markdown/content-format handling further down the stack.
*
* Spec-compliant with https://llmstxt.org/
*/
final class UrlSuffixMiddleware implements MiddlewareInterface
final readonly class UrlSuffixMiddleware implements MiddlewareInterface
{
public const REQUEST_ATTRIBUTE = 'llms_txt_format';
private const MARKDOWN_SUFFIX = '.md';
private const INDEX_SUFFIX = '/index.html.md';

public function __construct(
private SiteMatcher $siteMatcher,
) {}

public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
{
$uri = $request->getUri();
$path = $uri->getPath();

// Check for .md suffix
if (str_ends_with($path, self::INDEX_SUFFIX)) {
$isIndexSuffix = str_ends_with($path, self::INDEX_SUFFIX);
$isMarkdownSuffix = !$isIndexSuffix && str_ends_with($path, self::MARKDOWN_SUFFIX);

if (!$isIndexSuffix && !$isMarkdownSuffix) {
return $handler->handle($request);
}

// Site isn't resolved yet at this point in the middleware stack (this
// middleware runs before the site resolver on purpose), so the site
// has to be matched independently to read its Markdown setting.
if (!$this->isMarkdownEnabledForRequest($request)) {
return $handler->handle($request);
}

if ($isIndexSuffix) {
// /page/index.html.md -> /page/
$newPath = substr($path, 0, -\strlen(self::INDEX_SUFFIX) + 1);
$request = $this->rewriteRequest($request, $newPath);
} elseif (str_ends_with($path, self::MARKDOWN_SUFFIX)) {
} else {
// /page.md -> /page or /page/.md -> /page/
$newPath = substr($path, 0, -\strlen(self::MARKDOWN_SUFFIX));
// Handle /page/.md edge case
Expand All @@ -52,6 +75,22 @@ public function process(ServerRequestInterface $request, RequestHandlerInterface
return $handler->handle($request);
}

/**
* Whether Markdown output is enabled for the site matching this request.
*
* Defaults to true (enabled) if no site can be matched yet, matching the
* setting's own default and preserving prior behaviour for edge cases.
*/
private function isMarkdownEnabledForRequest(ServerRequestInterface $request): bool
{
$routeResult = $this->siteMatcher->matchRequest($request);
if (!$routeResult instanceof SiteRouteResult) {
return true;
}

return (bool)$routeResult->getSite()->getSettings()->get('llmsTxt.enableMarkdown', true);
}

/**
* Rewrite the request with new path and set format attribute.
*/
Expand Down
41 changes: 25 additions & 16 deletions Classes/Service/LlmsTxtGeneratorService.php
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public function getContentForSite(Site $site): string
$excludePages = $this->parseExcludePages($settings['excludePages'] ?? '');
$includeHidden = (bool)($settings['includeHidden'] ?? false);
$intro = trim((string)($settings['intro'] ?? ''));
$enableMarkdown = (bool)($settings['enableMarkdown'] ?? true);

$pages = $this->pageTreeService->getPages($site, $defaultLanguage, $excludePages, $includeHidden);

Expand All @@ -58,7 +59,7 @@ public function getContentForSite(Site $site): string

$apiKey = trim((string)($settings['apiKey'] ?? ''));

return $this->buildContent($site, $defaultLanguage, $pages, $baseUrl, $intro, $apiKey);
return $this->buildContent($site, $defaultLanguage, $pages, $baseUrl, $intro, $apiKey, $enableMarkdown);
}

/**
Expand Down Expand Up @@ -90,6 +91,7 @@ private function buildContent(
string $baseUrl,
string $intro,
string $apiKey,
bool $enableMarkdown,
): string {
$lines = [];

Expand All @@ -115,18 +117,21 @@ private function buildContent(
$lines[] = '**Generated:** ' . date('Y-m-d H:i:s');
$lines[] = '';

// Find an example page (first non-root page for realistic examples)
$examplePageUrl = $this->findExamplePageUrl($site, $sortedPages, $language);

// LLM-optimized content access section (spec-compliant with llmstxt.org)
$lines[] = '## LLM-Optimized Content Access';
$lines[] = '';
$lines[] = 'This site provides LLM-friendly Markdown output for all pages:';
$lines[] = '';
$lines[] = '### Markdown Format';
$lines[] = 'Append `.md` to any page URL to get plain Markdown with YAML frontmatter.';
$lines[] = '- **Example:** `' . $this->buildMarkdownUrl($examplePageUrl) . '`';
$lines[] = '';
// Omitted entirely when Markdown output is disabled for this site.
if ($enableMarkdown) {
// Find an example page (first non-root page for realistic examples)
$examplePageUrl = $this->findExamplePageUrl($site, $sortedPages, $language);

$lines[] = '## LLM-Optimized Content Access';
$lines[] = '';
$lines[] = 'This site provides LLM-friendly Markdown output for all pages:';
$lines[] = '';
$lines[] = '### Markdown Format';
$lines[] = 'Append `.md` to any page URL to get plain Markdown with YAML frontmatter.';
$lines[] = '- **Example:** `' . $this->buildMarkdownUrl($examplePageUrl) . '`';
$lines[] = '';
}

// Add authentication section if API key is configured
if ($apiKey !== '') {
Expand All @@ -140,7 +145,9 @@ private function buildContent(
$lines[] = '';
$lines[] = '**Query Parameter:**';
$lines[] = '```';
$lines[] = $baseUrl . '/page.md?api_key=<your-api-key>';
$lines[] = $enableMarkdown
? $baseUrl . '/page.md?api_key=<your-api-key>'
: $baseUrl . '/llms.txt?api_key=<your-api-key>';
$lines[] = '```';
$lines[] = '';
}
Expand Down Expand Up @@ -178,9 +185,11 @@ private function buildContent(
$lines[] = str_repeat(' ', $indent) . ' > ' . str_replace("\n", ' ', $summary);
}

// Add format access hints (spec-compliant .md suffix)
$mdUrl = $this->buildMarkdownUrl($pageUrl);
$lines[] = str_repeat(' ', $indent) . ' [Markdown](' . $mdUrl . ')';
// Add format access hints (spec-compliant .md suffix), unless disabled
if ($enableMarkdown) {
$mdUrl = $this->buildMarkdownUrl($pageUrl);
$lines[] = str_repeat(' ', $indent) . ' [Markdown](' . $mdUrl . ')';
}
$lines[] = '';
}

Expand Down
6 changes: 6 additions & 0 deletions Configuration/Sets/LlmsTxt/settings.definitions.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ settings:
label: 'Include Hidden Pages'
description: 'Include hidden pages in llms.txt generation'
category: llmstxt
llmsTxt.enableMarkdown:
type: bool
default: true
label: 'Enable Markdown output'
description: 'Serve the .md Markdown variant for pages and reference it from llms.txt. If disabled, .md URLs are left untouched (404) and no Markdown links are listed in llms.txt.'
category: llmstxt
llmsTxt.apiKey:
type: string
default: ''
Expand Down
12 changes: 12 additions & 0 deletions Documentation/Configuration.rst
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,18 @@ you can configure the extension in **Site Management > Settings**.
If enabled, hidden pages are also included in the llms.txt generation.
This can be useful for staging environments or preview purposes.

.. _confval-enableMarkdown:

.. confval:: llmsTxt.enableMarkdown

:type: boolean
:Default: true

If disabled, the ``.md`` Markdown variant is no longer served for any
page on this site: ``.md`` URLs are left untouched and resolve as a
normal 404 instead of being rewritten, and llms.txt no longer lists a
"Markdown Format" section or per-page Markdown links.

.. _confval-apiKey:

.. confval:: llmsTxt.apiKey
Expand Down