diff --git a/.github/workflows/link-check-weekly.yml b/.github/workflows/link-check-weekly.yml index b8def5566b..4b939673ff 100644 --- a/.github/workflows/link-check-weekly.yml +++ b/.github/workflows/link-check-weekly.yml @@ -44,6 +44,10 @@ jobs: run: make test 2>&1 | tee link-check.log env: CI: 'true' + # Crawl every built version (via sitemap-full.xml), not just latest. This job is + # non-blocking and files an issue on failure, so it is the safe place to surface the + # full-version backlog. PR CI stays latest-only until the backlog is cleared. + FULL_COVERAGE: 'true' - name: Upload the link-check log if: always() diff --git a/__tests__/crawler.test.js b/__tests__/crawler.test.js index 322a163bb9..8f59cc2d4b 100644 --- a/__tests__/crawler.test.js +++ b/__tests__/crawler.test.js @@ -22,7 +22,10 @@ test('Crawl the docs and execute tests', async () => { const varSkipList = ['{{end}}']; const liquidRegex = /\{%.*?%}/gs; const liquidSkipList = []; - const SITEMAP = 'sitemap.xml'; + // FULL_COVERAGE seeds the crawl from the all-versions sitemap (sitemap-full.xml) instead of the + // latest-only sitemap.xml. Used by the weekly job; leave off for PR CI until the backlog is fixed. + const fullCoverage = process.env.FULL_COVERAGE === 'true'; + const SITEMAP = fullCoverage ? 'sitemap-full.xml' : 'sitemap.xml'; const SITEMAP_URL = `${DOCS}/${SITEMAP}`; const USE_LC = [ { regex: inspectFilesRegExp(), processContent: true }, @@ -612,6 +615,7 @@ test('Crawl the docs and execute tests', async () => { await crawler.addRequests(urls); console.log(`Crawling the docs (${DOCS}) and executing tests.`); + console.log(`Coverage: ${fullCoverage ? 'FULL (all versions)' : 'latest only'} via ${SITEMAP}.`); console.log(`Localhost mode is ${isLocalHost ? 'ON' : 'OFF'}.`); console.log(`Validity tests on code blocks: ${validityTest.length ? validityTest.join(',') : 'none'}`); console.log(`Validity tests on files: ${validityTestFiles.length ? validityTestFiles.join(',') : 'none'}`); diff --git a/docusaurus.config.js b/docusaurus.config.js index ec4dc5bfe4..6d9dc4212d 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -411,6 +411,19 @@ export default async function createAsyncConfig() { }, }, plugins: [ + // Second sitemap instance for the internal link checker only. The preset sitemap above + // stays latest-only for SEO (sitemap.xml, the one advertised in robots.txt). This one has + // no version ignorePatterns, so it lists every built version at sitemap-full.xml. The + // crawler seeds from it when FULL_COVERAGE=true; it is not referenced in robots.txt. + [ + '@docusaurus/plugin-sitemap', + { + id: 'full', + filename: 'sitemap-full.xml', + lastmod: 'date', + ignorePatterns: [], + }, + ], 'docusaurus-plugin-sass', [ '@docusaurus/plugin-content-docs', diff --git a/package.json b/package.json index d7021ec290..9beb88428e 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "@docusaurus/core": "^3.10.1", "@docusaurus/faster": "^3.10.1", "@docusaurus/plugin-content-docs": "^3.10.1", + "@docusaurus/plugin-sitemap": "3.10.1", "@docusaurus/preset-classic": "^3.10.1", "@docusaurus/theme-common": "^3.10.1", "@docusaurus/theme-search-algolia": "^3.10.1", diff --git a/yarn.lock b/yarn.lock index 5c22c9a85f..d5c12670cc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -20109,6 +20109,7 @@ __metadata: "@docusaurus/faster": "npm:^3.10.1" "@docusaurus/module-type-aliases": "npm:^3.10.1" "@docusaurus/plugin-content-docs": "npm:^3.10.1" + "@docusaurus/plugin-sitemap": "npm:3.10.1" "@docusaurus/preset-classic": "npm:^3.10.1" "@docusaurus/theme-common": "npm:^3.10.1" "@docusaurus/theme-search-algolia": "npm:^3.10.1"