html: deep pages, text in other scripts, every element of the standard, and a byte order mark

CHANGELOG.md

@@ -139,6 +139,24 @@ because it turns other people's test suites red.
   antimeridian has its west edge greater than its east edge. GDAL opens a file
   whose bbox is wrong without a word, so a reader that relies on bbox needs a
   test of its own. Files made with the earlier settings keep their bytes.
+- **HTML: deep pages, text in other scripts, every element, and a byte order
+  mark.** `nesting` puts the content of a page inside up to 100 000 div
+  elements, each inside the last, with the label outside them. Readers part
+  ways early: lxml drops everything but the label from a page about 250
+  deep, Chromium keeps 511 levels and moves the deeper ones out, and jsdom,
+  parse5 and DOMPurify can run out of stack writing out a page from about
+  2 000 deep. `characters=international` adds paragraphs in Polish, Russian,
+  Japanese and Arabic, the Arabic right to left, with emoji, a letter written
+  as two code points and entities. `elements=every` adds a block that uses
+  every element of the HTML standard but script at least once, with all 22
+  kinds of input and small pictures and a sound written into the page, so
+  opening it fetches nothing - the Nu HTML Checker finds nothing wrong with
+  it, and pandoc's reader leaves out the header, the aside and the footer.
+  `bom` puts a byte order mark in front. A page with one is still HTML to
+  Chromium and to `file`, and plain text to the content sniffing the WHATWG
+  describes, which Go's `net/http` follows - so a filter that recognises
+  HTML by its first bytes can let it through. Files made with the earlier
+  settings keep their bytes.
 - **A length of time as a setting.** `duration`, `change_interval` and `keyframe_interval` take
   `10s`, `1m30s`, `1h`, `500ms` or `59.9s`, to the millisecond. A bare number
   is refused rather than read as seconds, and so is a length that does not

README.md

@@ -621,7 +621,7 @@ recipe. `tfg formats <id>` prints the allowed range or list for each:
 | `geojson` | `geometry`, `formatting`, `precision`, `altitude`, `vertices`, `holes`, `winding`, `antimeridian`, `unlocated`, `ids`, `bbox` |
 | `eml` | `body`, `text_encoding`, `charset`, `headers`, `recipients`, `line_endings`, `attachments`, `attachment_format`, `attachment_size`, `filename_style`, `message_depth`, `inline`, `inline_format`, `inline_size` |
 | `svg` | `width`, `height` |
-| `html` | `structure` |
+| `html` | `structure`, `nesting`, `characters`, `elements`, `bom` |
 | `yaml`, `toml` | none in this build - the document is a fixed shape, and a size is the only thing to ask for |
 
 ```

internal/format/htmlfile/blocks.go

@@ -0,0 +1,117 @@
+package htmlfile
+
+import (
+	// D11 promises the same bytes from the same seed, so a deliberate,
+	// reproducible generator is the product rather than a weakness. Nothing
+	// here ever makes a secret.
+	// nosemgrep: go.lang.security.audit.crypto.math_random.math-random-used
+	"math/rand/v2"
+	"strconv"
+
+	"github.com/donislawdev/TestingFilesGenerator/internal/core"
+)
+
+// The blocks that fill a page, split out of html.go on 2026-10-14 with the
+// settings. Nothing here changed in that move - these are the bytes every page
+// has been made of since 2026-08-01, and the golden hashes hold them there.
+
+// blocks builds the body. A natural record is one complete block element and
+// the closing record is a paragraph stretched to land the byte count.
+//
+// Whole blocks only, for the same reason Markdown writes whole blocks: a table
+// or a list cut in the middle still renders, and it says something other than
+// it meant.
+type blocks struct {
+	// tail is what the closing record ends with, which is the only thing the
+	// shape of the file changes down here.
+	tail string
+}
+
+// Shortest is the smallest closing record: an empty paragraph plus whatever
+// this shape closes after it.
+func (b blocks) Shortest() int64 { return int64(len(paraOpen) + len(b.tail)) }
+
+func (blocks) Append(dst []byte, rng *rand.Rand) []byte {
+	switch rng.IntN(5) {
+	case 0:
+		dst = append(dst, "<h2>"...)
+		dst = appendPhrase(dst, rng, 2+rng.IntN(3))
+		return append(dst, "</h2>\n"...)
+	case 1:
+		dst = append(dst, "<ul>\n"...)
+		for i, n := 0, 2+rng.IntN(3); i < n; i++ {
+			dst = append(dst, "<li>"...)
+			dst = appendPhrase(dst, rng, 3+rng.IntN(4))
+			dst = append(dst, "</li>\n"...)
+		}
+		return append(dst, "</ul>\n"...)
+	case 2:
+		dst = append(dst, "<table>\n<tr><th>id</th><th>field</th><th>value</th></tr>\n"...)
+		for i, n := 0, 2+rng.IntN(3); i < n; i++ {
+			dst = append(dst, "<tr><td>"...)
+			dst = strconv.AppendInt(dst, int64(i+1), 10)
+			dst = append(dst, "</td><td>"...)
+			dst = append(dst, words[rng.IntN(len(words))]...)
+			dst = append(dst, "</td><td>"...)
+			dst = append(dst, words[rng.IntN(len(words))]...)
+			dst = append(dst, "</td></tr>\n"...)
+		}
+		return append(dst, "</table>\n"...)
+	case 3:
+		// A real page carries entities, and an ampersand left raw is the
+		// classic way a document stops being well formed. The fixture carries
+		// one on purpose.
+		dst = append(dst, "<blockquote><p>"...)
+		dst = appendPhrase(dst, rng, 4+rng.IntN(4))
+		dst = append(dst, " &amp; "...)
+		dst = appendPhrase(dst, rng, 2)
+		return append(dst, "</p></blockquote>\n"...)
+	default:
+		dst = append(dst, paraOpen...)
+		dst = appendPhrase(dst, rng, 12+rng.IntN(12))
+		return append(dst, paraClose...)
+	}
+}
+
+// Discard has nothing to put back. A block carries no state from one to the
+// next, so throwing one away leaves no trace to undo.
+func (blocks) Discard() {}
+
+func (b blocks) AppendExact(dst []byte, rng *rand.Rand, n int64) []byte {
+	start := len(dst)
+	dst = append(dst, paraOpen...)
+	used := int64(len(dst)-start) + int64(len(b.tail))
+	dst = appendFiller(dst, n-used)
+	return append(dst, b.tail...)
+}
+
+func appendPhrase(dst []byte, rng *rand.Rand, n int) []byte {
+	for i := 0; i < n; i++ {
+		if i > 0 {
+			dst = append(dst, ' ')
+		}
+		dst = append(dst, words[rng.IntN(len(words))]...)
+	}
+	return dst
+}
+
+// appendFiller writes exactly n bytes of paragraph text out of readable words.
+//
+// It never emits an ampersand or an angle bracket, the two characters that
+// would have to be escaped - an escape would make the text longer than the
+// count asked for.
+// appendFiller stretches the closing paragraph to the byte.
+func appendFiller(dst []byte, n int64) []byte {
+	return core.AppendFiller(dst, words, n, nil)
+}
+
+// words is the filler vocabulary. English by default, like the rest of the text
+// group.
+var words = []string{
+	"account", "anchor", "banner", "browser", "caption", "content", "control",
+	"default", "element", "feature", "footer", "gallery", "handler", "header",
+	"heading", "inline", "layout", "listing", "margin", "marker", "message",
+	"module", "navigation", "notice", "option", "padding", "preview", "profile",
+	"section", "selector", "sidebar", "summary", "template", "toolbar",
+	"tooltip", "viewport", "widget", "wrapper",
+}

internal/format/htmlfile/catalogue.go

@@ -0,0 +1,154 @@
+package htmlfile
+
+// The catalogue a page carries when elements is every: every element the HTML
+// standard lists, each one at least once, in a page the Nu HTML Checker calls
+// conforming.
+//
+// The list was read from the standard itself on 2026-10-14 - the index "List of
+// elements", 115 names - rather than from memory, which would have missed
+// search, selectedcontent and the return of menu. The draft of this block went
+// through four readers before it came here (docs/HTML-2-2026-10-14.md section
+// 11.2): the checker found one error in it, a label around the new select whose
+// button is a second thing to label, and passes it now with nothing to say.
+// Chromium 152 builds every element, none of them unknown.
+//
+// One element is left out, and on purpose: script. A page that runs code is a
+// security fixture, and those are a separate, opt-in set (decision D3). Nothing
+// here reaches a network either - every picture, the sound and the frame are
+// written into the page as data, so opening the file fetches nothing.
+//
+// The text is plain English in ASCII, the ruby included, so this setting and
+// characters stay independent of each other: a page can carry every element
+// with no letter outside ASCII, and every script with nothing but paragraphs.
+
+const (
+	// picture is an 8 by 8 PNG of one colour, 74 bytes, and sound 16 samples of
+	// silence as an 8 bit WAV at 8 kHz, 60 bytes. Both were made once and are
+	// written here as they are, so the page is the same on every run.
+	picture = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEUlEQVR42mNwaDiAFTEMLQkAYvNgAQlPC90AAAAASUVORK5CYII="
+	sound   = "data:audio/wav;base64,UklGRjQAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YRAAAACAgICAgICAgICAgICAgICA"
+
+	// catalogueHead is what the catalogue puts in the head of a page. base
+	// names no address, so it changes nothing a link resolves to, and the icon
+	// is empty data so a browser asks nobody for one.
+	catalogueHead = `<base target="_self">` + "\n" +
+		`<link rel="icon" href="data:,">` + "\n" +
+		`<meta name="description" content="Every element of the HTML standard, each one at least once">` + "\n" +
+		"<style>\nmark { background: #ffeb3b }\n</style>\n"
+
+	catalogueBody = `<header>
+<hgroup><h1>Every element</h1><p>Every element of the HTML standard, each one at least once.</p></hgroup>
+<nav><ul><li><a href="#text">Text</a></li><li><a href="#tables">Tables</a></li><li><a href="#forms">Forms</a></li><li><a href="#media">Media</a></li></ul></nav>
+<search><form><label>Search <input type="search" name="q"></label> <button type="submit">Find</button></form></search>
+</header>
+<main>
+<article id="text">
+<h2>Text</h2>
+<p>Plain <a href="#forms">link</a>, <abbr title="HyperText Markup Language">HTML</abbr>, <b>bold</b>, <bdi>name</bdi>, <bdo dir="rtl">reversed</bdo>, <cite>a title</cite>, <code>code()</code>, <data value="42">forty two</data>, <del>removed</del>, <dfn>term</dfn>, <em>stressed</em>, <i>alternate</i>, <ins>added</ins>, <kbd>Ctrl</kbd>, <mark>marked</mark>, <q>quoted</q>, <s>no longer accurate</s>, <samp>output</samp>, <small>fine print</small>, <span>span</span>, <strong>important</strong>, H<sub>2</sub>O, x<sup>2</sup>, <time datetime="2026-10-14">14 October 2026</time>, <u>annotated</u>, <var>n</var>, long<wbr>word, a line<br>break.</p>
+<p><ruby>base<rp>(</rp><rt>annotation</rt><rp>)</rp></ruby> and <math><mi>x</mi><mo>=</mo><mn>2</mn></math> and <slot name="note">slot fallback</slot> and <tfg-note>a custom element</tfg-note>.</p>
+<h3>Third level</h3>
+<h4>Fourth level</h4>
+<h5>Fifth level</h5>
+<h6>Sixth level</h6>
+<blockquote cite="#text"><p>A quotation in a block.</p></blockquote>
+<pre><code>preformatted   text
+  keeps   its spaces</code></pre>
+<hr>
+<dl><dt>Term</dt><dd>Its description.</dd></dl>
+<ol start="3" reversed><li>three</li><li>two</li></ol>
+<menu><li><button type="button">Menu button</button></li></menu>
+<address>Example Street 1, Example City</address>
+<details open><summary>Summary of the details</summary><p>The details themselves.</p></details>
+<dialog open><p>A dialog that is open.</p></dialog>
+<section><h3>A section</h3><p>Inside a section.</p></section>
+<div>A div.</div>
+</article>
+<article id="tables">
+<h2>Tables</h2>
+<table>
+<caption>A table with every part</caption>
+<colgroup><col><col span="2"></colgroup>
+<thead><tr><th scope="col">id</th><th scope="col">field</th><th scope="col">value</th></tr></thead>
+<tbody><tr><td rowspan="2">1</td><td>first</td><td>alpha</td></tr><tr><td colspan="2">spans two columns</td></tr></tbody>
+<tfoot><tr><td colspan="3">footer row</td></tr></tfoot>
+</table>
+</article>
+<article id="forms">
+<h2>Forms</h2>
+<form>
+<fieldset>
+<legend>Every type of input</legend>
+<label>hidden <input type="hidden" name="hidden" value="h"></label>
+<label>text <input type="text" name="text"></label>
+<label>search <input type="search" name="search"></label>
+<label>tel <input type="tel" name="tel"></label>
+<label>url <input type="url" name="url"></label>
+<label>email <input type="email" name="email"></label>
+<label>password <input type="password" name="password"></label>
+<label>date <input type="date" name="date"></label>
+<label>month <input type="month" name="month"></label>
+<label>week <input type="week" name="week"></label>
+<label>time <input type="time" name="time"></label>
+<label>datetime-local <input type="datetime-local" name="datetime"></label>
+<label>number <input type="number" name="number"></label>
+<label>range <input type="range" name="range"></label>
+<label>color <input type="color" name="color"></label>
+<label>checkbox <input type="checkbox" name="checkbox"></label>
+<label>radio <input type="radio" name="radio"></label>
+<label>file <input type="file" name="file"></label>
+<input type="submit" value="submit">
+<input type="image" src="` + picture + `" alt="image button" width="8" height="8">
+<input type="reset" value="reset">
+<input type="button" value="button">
+</fieldset>
+<label>select <select name="select"><optgroup label="Group"><option>one</option><option>two</option></optgroup></select></label>
+<label for="custom">custom select</label> <select id="custom" name="custom"><button><selectedcontent></selectedcontent></button><option>first</option><option>second</option></select>
+<label>datalist <input name="choice" list="choices"></label><datalist id="choices"><option value="alpha"></option><option value="beta"></option></datalist>
+<label>textarea <textarea name="textarea">text</textarea></label>
+<output name="output">output</output>
+<label>progress <progress value="3" max="10">3 of 10</progress></label>
+<label>meter <meter value="0.6">60%</meter></label>
+<button type="submit">button</button>
+</form>
+</article>
+<article id="media">
+<h2>Media</h2>
+<figure><img src="` + picture + `" alt="a picture" width="8" height="8" usemap="#map"><figcaption>A picture with a map.</figcaption></figure>
+<map name="map"><area shape="rect" coords="0,0,8,8" href="#media" alt="the whole picture"></map>
+<picture><source srcset="` + picture + `" type="image/png"><img src="` + picture + `" alt="a picture with a source" width="8" height="8"></picture>
+<audio controls src="` + sound + `">an audio player</audio>
+<video controls width="64" height="48" poster="` + picture + `"><source src="data:video/mp4," type="video/mp4"><track kind="captions" src="data:text/vtt,WEBVTT" srclang="en" label="English">a video player</video>
+<canvas width="8" height="8">a canvas</canvas>
+<svg width="16" height="16" viewBox="0 0 16 16"><rect width="16" height="16" fill="#4080c0"/></svg>
+<object data="` + picture + `" type="image/png" width="8" height="8">an object</object>
+<embed src="` + picture + `" type="image/png" width="8" height="8">
+<iframe srcdoc="&lt;p&gt;A frame with its own document.&lt;/p&gt;" sandbox title="an inline frame"></iframe>
+</article>
+</main>
+<aside><p>An aside.</p></aside>
+<template><p>A template, never shown.</p></template>
+<noscript><p>Shown only without scripts.</p></noscript>
+<footer><p>Page footer.</p></footer>
+`
+)
+
+// pageOnly is the elements that live only in a page's skeleton or its head,
+// which a fragment has neither of.
+var pageOnly = []string{"base", "body", "head", "html", "link", "meta", "style", "title"}
+
+// catalogueHeadFor is what the catalogue adds to the head, nothing when there
+// is no catalogue or no head.
+func (s settings) catalogueHeadFor() string {
+	if !s.every || s.shape != structureDocument {
+		return ""
+	}
+	return catalogueHead
+}
+
+// catalogueBlocks is what the catalogue adds to the body.
+func (s settings) catalogueBlocks() string {
+	if !s.every {
+		return ""
+	}
+	return catalogueBody
+}

internal/format/htmlfile/characters.go

@@ -0,0 +1,38 @@
+package htmlfile
+
+// The paragraphs a page carries when characters is international, measured on
+// four readers on 2026-10-14 (docs/HTML-2-2026-10-14.md section 11.4).
+//
+// Polish, Russian and Japanese are written as they are, so the line can be
+// read here. The Arabic is escaped, because text written right to left turns
+// the line around in an editor, and so are the characters nobody can see: the
+// joiner inside the emoji sequence and the accent that makes a second code
+// point of the e. Written as they are, a change to either would look like no
+// change at all.
+//
+// Each script is a paragraph of its own with its own lang, the Arabic one with
+// dir="rtl", because that is how a page says which language a passage is in -
+// and the attribute is what a reader keys its fonts and its direction on.
+const (
+	arabic = "\u0647\u0630\u0627 \u0646\u0635 \u0628\u0627\u0644\u0644\u063A\u0629 \u0627\u0644\u0639\u0631\u0628\u064A\u0629."
+
+	international = `<p lang="pl">Zażółć gęślą jaźń.</p>` + "\n" +
+		`<p lang="ru">Это текст на русском языке.</p>` + "\n" +
+		`<p lang="ja">これは日本語のテキストです。</p>` + "\n" +
+		`<p lang="ar" dir="rtl">` + arabic + `</p>` + "\n" +
+		// The entities are the ones a converter most often gets wrong: the
+		// five that have to be escaped, a non-breaking space, two named
+		// symbols, and a number written in decimal and in hex, one of them past
+		// the basic plane.
+		`<p>Emoji ` + "\U0001F600" + ` and a sequence ` + "\U0001F469\u200D\U0001F4BB" + `, ` +
+		`a letter in two code points e` + "\u0301" + `, ` +
+		`and entities &amp; &lt; &gt; &quot; &nbsp; &copy; &euro; &#8364; &#x1F600;.</p>` + "\n"
+)
+
+// internationalBlocks is what the setting adds, nothing for ascii.
+func (s settings) internationalBlocks() string {
+	if !s.international {
+		return ""
+	}
+	return international
+}