From 5bd72ee883ac147bcb16b0d5de0d4fe8d5d67089 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Wed, 14 Oct 2026 14:38:43 +0200 Subject: [PATCH 1/2] html: pictures inside the page, with the settings eml pictures have inline, inline_format and inline_size write real pictures of any picture format into an HTML page as data: URLs, one to a line above the blocks, so opening the page fetches nothing. They are the keys an eml message already takes, with the same meaning: one declaration in a new package, picture, that both formats use, and a guard that every format declaring them declares them alike. Each picture is the file its format writes on its own with the seed of its place, streamed through base64 without being held in memory, and checked against its plan as it is written. The base64 is one line because the Nu HTML Checker calls a line break inside src an error. A page whose pictures add up past what an int64 can count is refused rather than miscounted. The structural check is told the three settings and judges every picture with its own format's check. What the details say comes from pages made by the program and read by eleven readers: Chromium shows every format but tiff, DOMPurify keeps them all, and lxml leaves out a picture past about 7.1 MB unless told to read huge trees. Files made without pictures keep their bytes. Two golden cases pin pages with pictures. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 ++ README.md | 2 +- internal/format/eml/eml.go | 11 +- internal/format/eml/inline.go | 102 +++-------- internal/format/eml/settings.go | 5 +- internal/format/htmlfile/catalogue.go | 23 +-- internal/format/htmlfile/html.go | 27 ++- internal/format/htmlfile/pictures.go | 172 ++++++++++++++++++ internal/format/htmlfile/settings.go | 14 +- internal/format/picture/picture.go | 146 +++++++++++++++ internal/guard/generatorbytes_test.go | 8 + internal/guard/htmlpictures_test.go | 154 ++++++++++++++++ internal/guard/htmlsettings_test.go | 27 ++- internal/guard/layers_test.go | 6 +- internal/guard/parity_test.go | 3 + internal/guard/testdata/generator-golden.json | 8 + internal/gui/text/locale/registry/en.json | 20 ++ internal/gui/text/locale/registry/pl.json | 4 + internal/gui/text/locale/said/en.json | 30 +++ internal/gui/text/locale/said/pl.json | 6 + internal/oracle/strict.py | 77 ++++++++ web/public/ar/formats/index.html | 15 ++ web/public/cs/formaty/index.html | 15 ++ web/public/de/formate/index.html | 15 ++ web/public/es/formatos/index.html | 15 ++ web/public/formats/index.html | 15 ++ web/public/fr/formats/index.html | 15 ++ web/public/hi/formats/index.html | 15 ++ web/public/id/format/index.html | 15 ++ web/public/it/formati/index.html | 15 ++ web/public/ja/formats/index.html | 15 ++ web/public/ko/formats/index.html | 15 ++ web/public/nl/formaten/index.html | 15 ++ web/public/pl/formaty/index.html | 15 ++ web/public/pt-br/formatos/index.html | 15 ++ web/public/ro/formate/index.html | 15 ++ web/public/ru/formats/index.html | 15 ++ web/public/th/formats/index.html | 15 ++ web/public/tr/bicimler/index.html | 15 ++ web/public/uk/formats/index.html | 15 ++ web/public/vi/dinh-dang/index.html | 15 ++ web/public/zh-hans/formats/index.html | 15 ++ web/public/zh-hant/formats/index.html | 15 ++ 43 files changed, 1065 insertions(+), 121 deletions(-) create mode 100644 internal/format/htmlfile/pictures.go create mode 100644 internal/format/picture/picture.go create mode 100644 internal/guard/htmlpictures_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c39cc220..ce5631b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -157,6 +157,17 @@ because it turns other people's test suites red. describes, which Go's `net/http` follows - so a filter that recognises HTML by its first bytes can let it through. Files made with the earlier settings keep their bytes. +- **HTML: pictures inside the page.** `inline`, `inline_format` and + `inline_size` - the same settings, with the same meaning, as the pictures + of an `eml` message - write real pictures of any picture format into the + page as `data:` URLs, each on a line of its own above the blocks, so + opening the page fetches nothing. Each picture is the very file its format + writes on its own. The Nu HTML Checker finds nothing wrong with a page of + any of the nine formats, Chromium shows every format but `tiff`, and + DOMPurify keeps them all. lxml, and BeautifulSoup on top of it, leave out + a picture past about 7.1 MB unless told to read huge trees - on Linux lxml + took over 20 seconds over such a page, and with a 15 MB picture it lost the + rest of the page as well. Files made without pictures keep their bytes. - **A length of time as a setting.** `duration`, `change_interval` and `keyframe_interval` take `10s`, `1m30s`, `1h`, `500ms` or `59.9s`, to the millisecond. A bare number is refused rather than read as seconds, and so is a length that does not diff --git a/README.md b/README.md index 1632f949..2acef44b 100644 --- a/README.md +++ b/README.md @@ -621,7 +621,7 @@ recipe. `tfg formats ` prints the allowed range or list for each: | `geojson` | `geometry`, `formatting`, `precision`, `altitude`, `vertices`, `holes`, `winding`, `antimeridian`, `unlocated`, `ids`, `bbox` | | `eml` | `body`, `text_encoding`, `charset`, `headers`, `recipients`, `line_endings`, `attachments`, `attachment_format`, `attachment_size`, `filename_style`, `message_depth`, `inline`, `inline_format`, `inline_size` | | `svg` | `width`, `height` | -| `html` | `structure`, `nesting`, `characters`, `elements`, `bom` | +| `html` | `structure`, `nesting`, `characters`, `elements`, `bom`, `inline`, `inline_format`, `inline_size` | | `yaml`, `toml` | none in this build - the document is a fixed shape, and a size is the only thing to ask for | ``` diff --git a/internal/format/eml/eml.go b/internal/format/eml/eml.go index 1de0aaf7..d96c697e 100644 --- a/internal/format/eml/eml.go +++ b/internal/format/eml/eml.go @@ -18,6 +18,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/archive" + "github.com/donislawdev/TestingFilesGenerator/internal/format/picture" ) const generatorVersion = "1" @@ -242,19 +243,15 @@ func forwarded(m *message) int { return 0 } -// describePictures adds the pictures to what reaches the manifest - always -// their settings, so a test reads the same keys whether there are any or not. +// describePictures adds the pictures to what reaches the manifest, in the +// words html uses for its own. func describePictures(props map[string]any, m *message) { var raw, encoded int64 for _, c := range m.pictures { raw += c.plan.Bytes encoded += c.encoded } - props[Inline] = len(m.pictures) - props[InlineFormat] = m.s.pictures.format - props[InlineSize] = m.s.pictures.size - props["inline_bytes"] = raw - props["inline_encoded_bytes"] = encoded + picture.Describe(props, m.s.pictures, len(m.pictures), raw, encoded) } func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { diff --git a/internal/format/eml/inline.go b/internal/format/eml/inline.go index e92e9bf7..472c0cf6 100644 --- a/internal/format/eml/inline.go +++ b/internal/format/eml/inline.go @@ -2,12 +2,11 @@ package eml import ( "fmt" - "strconv" "strings" "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" - "github.com/donislawdev/TestingFilesGenerator/internal/format/archive" + "github.com/donislawdev/TestingFilesGenerator/internal/format/picture" ) // Pictures in the text: real files of a picture format, carried beside the @@ -21,97 +20,40 @@ import ( // lists them when the HTML is alone, MimeKit never does. A system that limits // or counts attachments meets the same file three ways. -// Setting names, public under rule 10. +// Setting names, public under rule 10. The declaration and its reading are +// shared with html, which carries pictures inside the page (package picture). const ( - Inline = "inline" - InlineFormat = "inline_format" - InlineSize = "inline_size" + Inline = picture.Count + InlineFormat = picture.Format + InlineSize = picture.Size ) // picturesGroup is the block the picture settings sit in, on both surfaces. const picturesGroup = "Pictures in the text" -// defaultPictureFormat is what a picture is when nothing says - the format a -// mail program puts a pasted picture in. -const defaultPictureFormat = "png" - -// pictures is what the inline settings ask for. -type pictures struct { - count int - format string - size int64 -} - // pictureProperties are the settings of the pictures the text shows. func pictureProperties() []format.Property { - return []format.Property{ - { - Name: Inline, Kind: format.PropertyInt, - Min: 0, Max: archive.MaxMembers, - Default: "0", - Detail: "How many pictures the text shows. With html and alternative each one sits beside the HTML and the HTML points at it through cid:. " + - "With plain there is no HTML to point from, so they travel after the text as pictures to show inline. " + - "mailparser lists them among the attachments, and so does Python's email package when the HTML is alone. MimeKit never does.", - Group: picturesGroup, - }, - { - Name: InlineFormat, Kind: format.PropertyText, - Shape: "the id of a picture format, as tfg formats lists them", - Default: defaultPictureFormat, - Detail: "The format of the pictures. Any format whose files are pictures, such as png, jpg or gif.", - Group: picturesGroup, - }, - { - Name: InlineSize, Kind: format.PropertySize, - Default: archive.DefaultSizeText, - Detail: "How big each picture is. Like an attached file it travels in base64, about a third larger.", - Group: picturesGroup, - }, - } -} - -// readPictures is the three settings as numbers. Whether the format is a -// picture is asked in Plan, where the registry is complete - this also runs -// while the formats are still registering, to work out the minimum. -func readPictures(values map[string]string) (pictures, error) { - n, err := strconv.Atoi(values[Inline]) - if err != nil { - return pictures{}, core.Defect(err) - } - size, err := core.ParseSize(values[InlineSize]) - if err != nil { - return pictures{}, core.Defect(err) - } - return pictures{count: n, format: values[InlineFormat], size: size}, nil + return picture.Properties(picturesGroup, picture.Details{ + Count: "How many pictures the text shows. With html and alternative each one sits beside the HTML and the HTML points at it through cid:. " + + "With plain there is no HTML to point from, so they travel after the text as pictures to show inline. " + + "mailparser lists them among the attachments, and so does Python's email package when the HTML is alone. MimeKit never does.", + Format: "The format of the pictures. Any format whose files are pictures, such as png, jpg or gif.", + Size: "How big each picture is. Like an attached file it travels in base64, about a third larger.", + }) } // pictureFormat is the descriptor of the format the pictures are, or the // refusal when it is not a picture. func pictureFormat(id string) (format.Descriptor, error) { - desc, err := format.Get(id) - if err != nil { - return format.Descriptor{}, err - } - if strings.HasPrefix(desc.MediaType, "image/") { - return desc, nil + desc, ok, err := picture.Of(id) + if err != nil || ok { + return desc, err } return format.Descriptor{}, &format.PropertyValueError{ Format: "eml", Key: InlineFormat, Value: id, Reason: core.Says("eml.APictureInTheTextIs", "a picture in the text has to be a file a mail program can show as a picture"), - Remedy: core.Says("eml.UseOneOfThePictureFormats", "Use one of: %s.", core.A("Formats", strings.Join(pictureFormats(), ", "))), - } -} - -// pictureFormats are the formats of this build whose files are pictures, from -// the media type each declares rather than from a list kept here. -func pictureFormats() []string { - var out []string - for _, d := range format.All() { - if strings.HasPrefix(d.MediaType, "image/") { - out = append(out, d.ID) - } + Remedy: core.Says("eml.UseOneOfThePictureFormats", "Use one of: %s.", core.A("Formats", strings.Join(picture.Formats(), ", "))), } - return out } // addPictures plans the pictures, after the attached files, with seeds that @@ -121,14 +63,14 @@ func pictureFormats() []string { // value the setting cannot take is a mistake either way, and accepting it in // silence would hand back a message the recipe did not describe. func (p *planner) addPictures(s settings) ([]child, error) { - desc, err := pictureFormat(s.pictures.format) - if err != nil || s.pictures.count == 0 { + desc, err := pictureFormat(s.pictures.Format) + if err != nil || s.pictures.Count == 0 { return nil, err } - out := make([]child, 0, s.pictures.count) + out := make([]child, 0, s.pictures.Count) first := len(p.out) - for i := 0; i < s.pictures.count; i++ { - c, err := p.plan(desc, s.pictures.size, first+i) + for i := 0; i < s.pictures.Count; i++ { + c, err := p.plan(desc, s.pictures.Size, first+i) if err != nil { return nil, err } diff --git a/internal/format/eml/settings.go b/internal/format/eml/settings.go index 9c6d53bb..501e5e24 100644 --- a/internal/format/eml/settings.go +++ b/internal/format/eml/settings.go @@ -6,6 +6,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/archive" + "github.com/donislawdev/TestingFilesGenerator/internal/format/picture" ) // Setting names. Public names under rule 10, so each is spelled once. @@ -110,7 +111,7 @@ type settings struct { // messages deep the message goes, itself counted. recipients int depth int - pictures pictures + pictures picture.Settings } // international says whether the text carries the line in Polish (and in @@ -174,7 +175,7 @@ func parse(props map[string]string) (settings, error) { if s.depth, err = strconv.Atoi(values[MessageDepth]); err != nil { return settings{}, core.Defect(err) } - s.pictures, err = readPictures(values) + s.pictures, err = picture.Parse(values) return s, err } diff --git a/internal/format/htmlfile/catalogue.go b/internal/format/htmlfile/catalogue.go index d08d7d7e..67950605 100644 --- a/internal/format/htmlfile/catalogue.go +++ b/internal/format/htmlfile/catalogue.go @@ -22,11 +22,12 @@ package htmlfile // with no letter outside ASCII, and every script with nothing but paragraphs. const ( - // picture is an 8 by 8 PNG of one colour, 74 bytes, and sound 16 samples of - // silence as an 8 bit WAV at 8 kHz, 60 bytes. Both were made once and are - // written here as they are, so the page is the same on every run. - picture = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEUlEQVR42mNwaDiAFTEMLQkAYvNgAQlPC90AAAAASUVORK5CYII=" - sound = "data:audio/wav;base64,UklGRjQAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YRAAAACAgICAgICAgICAgICAgICA" + // smallPicture is an 8 by 8 PNG of one colour, 74 bytes, and sound 16 + // samples of silence as an 8 bit WAV at 8 kHz, 60 bytes. Both were made + // once and are written here as they are, so the page is the same on every + // run. + smallPicture = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEUlEQVR42mNwaDiAFTEMLQkAYvNgAQlPC90AAAAASUVORK5CYII=" + sound = "data:audio/wav;base64,UklGRjQAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YRAAAACAgICAgICAgICAgICAgICA" // catalogueHead is what the catalogue puts in the head of a page. base // names no address, so it changes nothing a link resolves to, and the icon @@ -97,7 +98,7 @@ const ( - + @@ -113,15 +114,15 @@ const (

Media

-
a picture
A picture with a map.
+
a picture
A picture with a map.
the whole picture -a picture with a source +a picture with a source - + a canvas -an object - +an object +
diff --git a/internal/format/htmlfile/html.go b/internal/format/htmlfile/html.go index 0928f860..cfb75473 100644 --- a/internal/format/htmlfile/html.go +++ b/internal/format/htmlfile/html.go @@ -56,9 +56,9 @@ func init() { // than a comment. Label: format.LabelVisible, Oracle: "python-html", - // Pictures inside the page, style sheets and a page on one line come - // later (docs/HTML-2-2026-10-14.md section 8). Declaring only what is - // here is what makes a recipe asking for them fail loudly instead of + // Style sheets and a page on one line come later + // (docs/HTML-2-2026-10-14.md section 8). Declaring only what is here + // is what makes a recipe asking for them fail loudly instead of // quietly producing something else. Properties: properties(), GeneratorVersion: generatorVersion, @@ -73,6 +73,9 @@ type memo struct { labelLine string // the visible heading, empty when absent seed uint64 settings settings + pictures []pictureLine + // pictureBytes is what the lines of the pictures take, together. + pictureBytes int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -81,7 +84,11 @@ func (generator) Plan(r format.Request) (format.Plan, error) { return format.Plan{}, err } - min := minimumFor(s) + pictures, pictureBytes, err := planPictures(r, s.pictures) + if err != nil { + return format.Plan{}, err + } + min := minimumFor(s) + pictureBytes if r.Bytes < min { return format.Plan{}, belowMinimum(s, r.Bytes, min) } @@ -107,6 +114,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { for k, v := range s.manifest() { p.Properties[k] = v } + describePictures(p.Properties, s.pictures, pictures) if s.every && s.shape == structureFragment { // The setting says every element, and a fragment cannot hold eight of // them. Saying so here is what keeps the manifest from claiming a @@ -118,7 +126,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { }) } - m := memo{seed: r.Seed, head: s.prologue(), settings: s} + m := memo{seed: r.Seed, head: s.prologue(), settings: s, pictures: pictures, pictureBytes: pictureBytes} if r.Label { var note *format.Note m.head, m.labelLine, note = labelledHead(s, r, min) @@ -136,6 +144,10 @@ func (generator) Plan(r format.Request) (format.Plan, error) { func belowMinimum(s settings, requested, min int64) error { reason := core.Says("html.MinimumPage", "a page holds a head, a body and whole blocks, and one of each needs that much") switch { + case s.pictures.Count > 0: + // The pictures are what a tester raises the size for, so they are what + // the reason names, whatever else was asked. + reason = core.Says("html.MinimumWithPictures", "the file holds whole blocks and every picture in base64, which is about a third larger than the picture itself, and together they need that much") case s.adds(): reason = core.Says("html.MinimumWithSettings", "the file holds whole blocks and what %s, %s, %s and %s add to them, and together they need that much", core.A("Nesting", core.LabelTerm(settingNesting)), core.A("Characters", core.LabelTerm(settingCharacters)), @@ -195,9 +207,12 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { if err := core.WriteAll(w, []byte(head)); err != nil { return err } + if err := writePictures(ctx, w, m.pictures); err != nil { + return err + } rng := core.NewRand(m.seed) - return core.FillRecords(ctx, w, rng, p.Bytes-int64(len(head)), m.settings.blocks()) + return core.FillRecords(ctx, w, rng, p.Bytes-int64(len(head))-m.pictureBytes, m.settings.blocks()) } // minimumBytes is the smallest whole page, which is what the registry declares. diff --git a/internal/format/htmlfile/pictures.go b/internal/format/htmlfile/pictures.go new file mode 100644 index 00000000..2e16a48b --- /dev/null +++ b/internal/format/htmlfile/pictures.go @@ -0,0 +1,172 @@ +package htmlfile + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "strings" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/picture" +) + +// Pictures inside the page: real files of a picture format, each in a data: +// URL (RFC 2397) on a line of its own, the way a page saved as one file +// carries them. The settings are the ones a mail message has, with the same +// names and the same meaning (package picture). +// +// Measured on 2026-10-14 on pages from this program (docs/HTML-3-2026-10-14.md +// section 9), and this is what the details below say: Chromium shows every +// picture format but tiff, the Nu HTML Checker calls a line break inside src an +// error - so the base64 is one line - and lxml keeps no attribute past ten +// million characters unless told to read huge trees. + +// picturesGroup is the block the picture settings sit in, on both surfaces. +const picturesGroup = "Pictures in the page" + +// maxPage is the largest page this format plans with pictures in it. Far above +// anything a disk holds, and far enough below the int64 every size is counted +// in that adding the pictures up cannot wrap round - eml's maxMessage, for its +// reason. +const maxPage = int64(1) << 60 + +// pictureClose ends the line a picture sits on. +const pictureClose = `">

` + "\n" + +// pictureProperties are the settings of the pictures the page shows. +func pictureProperties() []format.Property { + return picture.Properties(picturesGroup, picture.Details{ + Count: "How many pictures the page shows, each one a real file of its format written into the page as a data: URL, so opening the page fetches nothing. " + + "They sit above the blocks, one to a line, and each says which it is in its alt text.", + Format: "The format of the pictures. Any format whose files are pictures, such as png, jpg or gif. Chromium shows every one of them but tiff.", + Size: "How big each picture is. Inside the page it is base64 on one line, about a third larger. " + + "lxml, and BeautifulSoup on top of it, leave out a picture past about 7.1 MB unless told to read huge trees. " + + "On Linux lxml took over 20 seconds over such a page, and with a 15 MB picture it lost the rest of the page as well.", + }) +} + +// pictureLine is one picture, planned: the file, and the line it sits on up +// to where the base64 starts. +type pictureLine struct { + desc format.Descriptor + plan format.Plan + open string +} + +// bytes is what the line takes in the page. +func (p pictureLine) bytes() int64 { + return int64(len(p.open)) + base64Length(p.plan.Bytes) + int64(len(pictureClose)) +} + +// planPictures plans the pictures, each with the seed of its place (RC4), so +// asking for one more changes none of the others. The format is asked whether +// it is a picture even when none are asked for: a value the setting cannot +// take is a mistake either way, and eml refuses it the same way. +func planPictures(r format.Request, s picture.Settings) ([]pictureLine, int64, error) { + desc, ok, err := picture.Of(s.Format) + if err != nil { + return nil, 0, err + } + if !ok { + return nil, 0, &format.PropertyValueError{ + Format: "html", Key: picture.Format, Value: s.Format, + Reason: core.Says("html.APictureInThePageIs", "a picture in the page has to be a file of a picture format"), + Remedy: core.Says("html.UseOneOfThePictureFormats", "Use one of: %s.", core.A("Formats", strings.Join(picture.Formats(), ", "))), + } + } + out := make([]pictureLine, 0, s.Count) + var sum int64 + for i := 0; i < s.Count; i++ { + cp, err := desc.Generator.Plan(format.Request{Bytes: s.Size, Seed: core.FileSeed(r.Seed, i), Label: r.Label}) + if err != nil { + return nil, 0, core.Refuse(core.Says("html.ThePictureCannotBe", "html: the %s picture cannot be made: %w", + core.A("Format", desc.ID), core.A("Err", err))) + } + if cp.Bytes > maxPage { + return nil, 0, pageTooLarge(cp.Bytes) + } + line := pictureLine{desc: desc, plan: cp, open: pictureOpen(desc, i+1)} + if sum += line.bytes(); sum > maxPage { + return nil, 0, pageTooLarge(sum) + } + out = append(out, line) + } + return out, sum, nil +} + +// pictureOpen is a picture's line up to its base64. The alt text is the name +// the picture has in a mail message, inline_0001.png, so a tester reading +// either finds the same picture by the same name. +func pictureOpen(desc format.Descriptor, n int) string { + name := fmt.Sprintf("inline_%04d%s", n, desc.Extension) + return `

` + name + `inline_([0-9]{4,5})(\.[a-z]+)

\n`) + +// TestAPictureInAPageIsTheFileItsFormatWritesAlone is the claim of inline. +// +// For every picture format of this build, from the registry rather than from a +// list here: the page carries as many pictures as were ordered, each named and +// typed as its format, each on one line, and each the very bytes the format +// writes on its own with the seed of its place - so a picture is a real file a +// tester can cut out of the page, not something that only looks like one. +func TestAPictureInAPageIsTheFileItsFormatWritesAlone(t *testing.T) { + formats := pic.Formats() + if len(formats) == 0 { + t.Fatal("this build has no picture format, so there is nothing a page could carry") + } + for _, id := range formats { + desc, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + props := map[string]string{"inline": "2", "inline_format": id, "inline_size": "2kb"} + body := writeEncoded(t, "html", 65536, props) + found := htmlPictureLine.FindAllSubmatch(body, -1) + if len(found) != 2 { + t.Errorf("%s: inline=2 was ordered and the page carries %d pictures on lines of their own", id, len(found)) + continue + } + if n := bytes.Count(body, []byte(`inline_`)); n != len(found) { + t.Errorf(inline_([0-9]{4,5})(\.[a-z]+)

\n') + + +def html_size(word): + """A size as a recipe writes it, in units of 1024 or as a byte count.""" + match = re.fullmatch(r"([0-9]{1,15})(b|kb|mb|gb)?", word.lower()) + if not match: + fail(f"the html check was told inline_size={word!r}, which is not a size this tool takes") + return int(match.group(1)) * {None: 1, "b": 1, "kb": 1024, "mb": 1024 ** 2, "gb": 1024 ** 3}[match.group(2)] + + +def html_pictures(text, settings): + """The pictures inline ordered: as many as were asked, numbered in order, + each of the format and size asked and each a file its own check accepts. + + TOLD, never counted from the page and believed, for the reason the shape + check gives. The catalogue of every element carries pictures of its own, + and they are told apart by their alt text. + """ + count = settings.get("inline", "0") + if not re.fullmatch(r"[0-9]{1,5}", count) or int(count) > 10000: + fail(f"the html check was told inline={count!r}, which is not a whole number from 0 to 10000") + kind = settings.get("inline_format", "png") + if kind not in HTML_PICTURE_TYPES: + fail(f"the html check was told inline_format={kind!r}, which is not a picture format") + size = html_size(settings.get("inline_size", "8kb")) + ext, media = HTML_PICTURE_TYPES[kind] + found = HTML_PICTURE.findall(text) + if len(found) != int(count): + fail(f"inline={count} was ordered and the page carries {len(found)} pictures") + if "base64," in text and text.count('inline_') != len(found): + fail(الإعدادات التي تقبلها كل صيغة bom صحيح أو خاطئ + + + inline + 0 - 10000 + + + + inline_format + معرّف صيغة صورة، كما يسردها tfg formats + + + + inline_size + حجم مثل 2mb + ico width diff --git a/web/public/cs/formaty/index.html b/web/public/cs/formaty/index.html index 2e21f0b0..deee58d5 100644 --- a/web/public/cs/formaty/index.html +++ b/web/public/cs/formaty/index.html @@ -677,6 +677,21 @@

Nastavení, která každý formát přijímá

bom pravda nebo nepravda + + + inline + 0 - 10000 + + + + inline_format + id obrázkového formátu tak, jak je vypisuje tfg formats + + + + inline_size + velikost, například 2mb + ico width diff --git a/web/public/de/formate/index.html b/web/public/de/formate/index.html index 0f941f39..f8164358 100644 --- a/web/public/de/formate/index.html +++ b/web/public/de/formate/index.html @@ -679,6 +679,21 @@

Einstellungen, die jedes Format kennt

bom wahr oder falsch + + + inline + 0 - 10000 + + + + inline_format + die ID eines Bildformats, wie tfg formats sie auflistet + + + + inline_size + eine Größe wie 2mb + ico width diff --git a/web/public/es/formatos/index.html b/web/public/es/formatos/index.html index 308389e0..7f2b432a 100644 --- a/web/public/es/formatos/index.html +++ b/web/public/es/formatos/index.html @@ -680,6 +680,21 @@

Ajustes que admite cada formato

bom verdadero o falso + + + inline + 0 - 10000 + + + + inline_format + el id de un formato de imagen, tal como lo lista tfg formats + + + + inline_size + un tamaño como 2mb + ico width diff --git a/web/public/formats/index.html b/web/public/formats/index.html index b3deab86..f110ce72 100644 --- a/web/public/formats/index.html +++ b/web/public/formats/index.html @@ -678,6 +678,21 @@

Settings each format accepts

bom true or false + + + inline + 0 - 10000 + + + + inline_format + the id of a picture format, as tfg formats lists them + + + + inline_size + a size such as 2mb + ico width diff --git a/web/public/fr/formats/index.html b/web/public/fr/formats/index.html index c927fb0d..4aa62344 100644 --- a/web/public/fr/formats/index.html +++ b/web/public/fr/formats/index.html @@ -680,6 +680,21 @@

Réglages que chaque format accepte

bom vrai ou faux + + + inline + 0 - 10000 + + + + inline_format + l'identifiant d'un format d'image, comme le liste tfg formats + + + + inline_size + une taille comme 2mb + ico width diff --git a/web/public/hi/formats/index.html b/web/public/hi/formats/index.html index cb6a8500..243bde49 100644 --- a/web/public/hi/formats/index.html +++ b/web/public/hi/formats/index.html @@ -678,6 +678,21 @@

हर फ़ॉर्मैट की स्वीकार की जा bom सही या गलत + + + inline + 0 - 10000 + + + + inline_format + किसी चित्र फ़ॉर्मैट का id, जैसा tfg formats सूचीबद्ध करता है + + + + inline_size + 2mb जैसा आकार + ico width diff --git a/web/public/id/format/index.html b/web/public/id/format/index.html index 21f6d1a0..d0b7ac63 100644 --- a/web/public/id/format/index.html +++ b/web/public/id/format/index.html @@ -678,6 +678,21 @@

Pengaturan yang diterima setiap format

bom benar atau salah + + + inline + 0 - 10000 + + + + inline_format + id sebuah format gambar, seperti yang didaftar tfg formats + + + + inline_size + ukuran seperti 2mb + ico width diff --git a/web/public/it/formati/index.html b/web/public/it/formati/index.html index 29107970..74c69ad8 100644 --- a/web/public/it/formati/index.html +++ b/web/public/it/formati/index.html @@ -680,6 +680,21 @@

Impostazioni che ogni formato accetta

bom vero o falso + + + inline + 0 - 10000 + + + + inline_format + l'id di un formato di immagine, come lo elenca tfg formats + + + + inline_size + una dimensione come 2mb + ico width diff --git a/web/public/ja/formats/index.html b/web/public/ja/formats/index.html index 26233d85..540680fd 100644 --- a/web/public/ja/formats/index.html +++ b/web/public/ja/formats/index.html @@ -670,6 +670,21 @@

形式ごとに受け付ける設定

bom 真または偽 + + + inline + 0 - 10000 + + + + inline_format + tfg formatsが一覧表示する画像形式のID + + + + inline_size + 2mbのようなサイズ + ico width diff --git a/web/public/ko/formats/index.html b/web/public/ko/formats/index.html index f64fb353..bd578940 100644 --- a/web/public/ko/formats/index.html +++ b/web/public/ko/formats/index.html @@ -671,6 +671,21 @@

각 형식이 받는 설정

bom 참 또는 거짓 + + + inline + 0 - 10000 + + + + inline_format + tfg formats가 나열하는 이미지 형식의 id + + + + inline_size + 2mb와 같은 크기 + ico width diff --git a/web/public/nl/formaten/index.html b/web/public/nl/formaten/index.html index 9dc97913..5e0076f1 100644 --- a/web/public/nl/formaten/index.html +++ b/web/public/nl/formaten/index.html @@ -680,6 +680,21 @@

Instellingen die elk formaat accepteert

bom waar of onwaar + + + inline + 0 - 10000 + + + + inline_format + de id van een afbeeldingsformaat, zoals tfg formats ze opsomt + + + + inline_size + een grootte zoals 2mb + ico width diff --git a/web/public/pl/formaty/index.html b/web/public/pl/formaty/index.html index 32104209..a2615714 100644 --- a/web/public/pl/formaty/index.html +++ b/web/public/pl/formaty/index.html @@ -678,6 +678,21 @@

Ustawienia, które przyjmuje każdy format

bom prawda albo fałsz + + + inline + 0 - 10000 + + + + inline_format + identyfikator formatu obrazka, tak jak wypisuje go tfg formats + + + + inline_size + rozmiar, na przykład 2mb + ico width diff --git a/web/public/pt-br/formatos/index.html b/web/public/pt-br/formatos/index.html index f3aca110..0af97539 100644 --- a/web/public/pt-br/formatos/index.html +++ b/web/public/pt-br/formatos/index.html @@ -679,6 +679,21 @@

Configurações que cada formato aceita

bom verdadeiro ou falso + + + inline + 0 - 10000 + + + + inline_format + o id de um formato de imagem, como o tfg formats lista + + + + inline_size + um tamanho como 2mb + ico width diff --git a/web/public/ro/formate/index.html b/web/public/ro/formate/index.html index 54acf886..dd595dbe 100644 --- a/web/public/ro/formate/index.html +++ b/web/public/ro/formate/index.html @@ -679,6 +679,21 @@

Setările pe care le acceptă fiecare format

bom adevărat sau fals + + + inline + 0 - 10000 + + + + inline_format + id-ul unui format de imagine, așa cum îl listează tfg formats + + + + inline_size + o dimensiune precum 2mb + ico width diff --git a/web/public/ru/formats/index.html b/web/public/ru/formats/index.html index 5c5cc463..77c31f3c 100644 --- a/web/public/ru/formats/index.html +++ b/web/public/ru/formats/index.html @@ -678,6 +678,21 @@

Настройки, которые принимает каждый форм bom истина или ложь + + + inline + 0 - 10000 + + + + inline_format + id формата изображения в том виде, как его выводит tfg formats + + + + inline_size + размер, например 2mb + ico width diff --git a/web/public/th/formats/index.html b/web/public/th/formats/index.html index 69ae4f1a..abf61438 100644 --- a/web/public/th/formats/index.html +++ b/web/public/th/formats/index.html @@ -676,6 +676,21 @@

การตั้งค่าที่แต่ละรูปแบบร bom จริงหรือเท็จ + + + inline + 0 - 10000 + + + + inline_format + รหัสของรูปแบบรูปภาพ ตามที่ tfg formats แสดง + + + + inline_size + ขนาด เช่น 2mb + ico width diff --git a/web/public/tr/bicimler/index.html b/web/public/tr/bicimler/index.html index 643f8346..861d3ca2 100644 --- a/web/public/tr/bicimler/index.html +++ b/web/public/tr/bicimler/index.html @@ -678,6 +678,21 @@

Her biçimin kabul ettiği ayarlar

bom doğru veya yanlış + + + inline + 0 - 10000 + + + + inline_format + tfg formats'ın listelediği gibi bir görüntü biçiminin kimliği + + + + inline_size + 2mb gibi bir boyut + ico width diff --git a/web/public/uk/formats/index.html b/web/public/uk/formats/index.html index 4ece25dc..352fe8b9 100644 --- a/web/public/uk/formats/index.html +++ b/web/public/uk/formats/index.html @@ -678,6 +678,21 @@

Налаштування, які приймає кожен формат

bom істина або хибність + + + inline + 0 - 10000 + + + + inline_format + id формату зображення в тому вигляді, як його виводить tfg formats + + + + inline_size + розмір, наприклад 2mb + ico width diff --git a/web/public/vi/dinh-dang/index.html b/web/public/vi/dinh-dang/index.html index cd67bece..ee8e5b30 100644 --- a/web/public/vi/dinh-dang/index.html +++ b/web/public/vi/dinh-dang/index.html @@ -676,6 +676,21 @@

Các thiết lập mỗi định dạng nhận

bom đúng hoặc sai + + + inline + 0 - 10000 + + + + inline_format + id của một định dạng hình ảnh, như tfg formats liệt kê + + + + inline_size + kích thước như 2mb + ico width diff --git a/web/public/zh-hans/formats/index.html b/web/public/zh-hans/formats/index.html index 7672855e..34752965 100644 --- a/web/public/zh-hans/formats/index.html +++ b/web/public/zh-hans/formats/index.html @@ -670,6 +670,21 @@

每种格式接受的设置

bom 真或假 + + + inline + 0 - 10000 + + + + inline_format + 图片格式的 id，与 tfg formats 列出的一致 + + + + inline_size + 形如 2mb 的大小 + ico width diff --git a/web/public/zh-hant/formats/index.html b/web/public/zh-hant/formats/index.html index d05bc8ab..89629c61 100644 --- a/web/public/zh-hant/formats/index.html +++ b/web/public/zh-hant/formats/index.html @@ -670,6 +670,21 @@

每種格式接受的設定

bom 真或假 + + + inline + 0 - 10000 + + + + inline_format + 圖片格式的 id，與 tfg formats 列出的相同 + + + + inline_size + 形如 2mb 的大小 + ico width From 8f55eba5e0ebe86bc7752bb80e32a041842e549f Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Wed, 14 Oct 2026 15:13:26 +0200 Subject: [PATCH 2/2] Stop a container between the files it plans, stop webp after a stop, and give a broken picture a verdict A container plans every file inside it from inside its own Plan, and the engine asked for a stop only between one file and the next. Ten thousand small pngs in a page took 72 s to plan, and as long in a mail message, a zip and a tar.gz, with Ctrl+C waiting for the last of them. A generator may now also be a format.ContextPlanner: html, eml, zip and tar.gz plan their files through format.PlanWith, which asks the context before each one, and the engine hands its context in. A stop comes back as the stop itself rather than as a file that cannot be made, so a window marks no box for it. The request keeps no context, since a context is no part of what a plan is keyed by. The crash guard now forbids format.PlanWith outside the wrapper too, since it reaches a generator. webp noticed a stop every 64 rows and still coded the picture to its last row, five seconds for 600 MB. The row callback now ends the stream. The structural check of a page caught only the exit a picture's own check ends with, so a picture that stopped that check part way ended it with a traceback. Any exception there is now a verdict on the picture. Bytes of runs that are not stopped are unchanged. New guards hold each of the three, and each was seen to go red with its fix taken out. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 +++ internal/engine/crash.go | 7 +- internal/engine/drawsizes.go | 5 +- internal/engine/plantarget.go | 2 +- internal/format/eml/attach.go | 12 +-- internal/format/eml/eml.go | 10 ++- internal/format/format.go | 39 ++++++++ internal/format/htmlfile/html.go | 9 +- internal/format/htmlfile/pictures.go | 8 +- internal/format/targz/targz.go | 15 ++-- internal/format/webp/picture.go | 3 + internal/format/webp/vp8l.go | 8 +- internal/format/zip/children.go | 7 +- internal/format/zip/zip.go | 9 +- internal/guard/containerstop_test.go | 122 ++++++++++++++++++++++++++ internal/guard/generatorcrash_test.go | 8 +- internal/guard/htmlpictures_test.go | 32 +++++++ internal/guard/picturestop_test.go | 69 +++++++++++++++ internal/oracle/strict.py | 9 +- 19 files changed, 354 insertions(+), 32 deletions(-) create mode 100644 internal/guard/containerstop_test.go create mode 100644 internal/guard/picturestop_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ce5631b3..6f814876 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -277,6 +277,18 @@ because it turns other people's test suites red. ### Fixed +- **Ctrl+C waited for a file that holds thousands of files to be planned.** + A zip, a tar.gz, a mail message or a page with ten thousand small pictures + inside took over a minute to plan, and a run stopped in that minute went + on until the end of it - in CI long enough for the grace period to run + out. Planning now stops between one file inside and the next, and says + it stopped rather than that a file could not be made. + +- **A stopped run went on writing a WebP picture to its last row.** A + 600 MB picture took five more seconds to stop. It now stops within a few + rows, as the other picture formats do. Files that are not stopped keep + their bytes. + - **Choosing another tool on the Tools tab left the boxes in the wrong place.** The section kept the height of the tool chosen before, so a setting could stand under it with its list hidden, or an empty band was diff --git a/internal/engine/crash.go b/internal/engine/crash.go index 9037fe89..68f1e17d 100644 --- a/internal/engine/crash.go +++ b/internal/engine/crash.go @@ -80,7 +80,10 @@ const ( // container planning its children calls them from inside its own Plan, so a // child crashing is caught here too - and named as the container, because the // container is what this was asked to plan. -func planWithoutCrashing(desc format.Descriptor, r format.Request) (p format.Plan, err error) { +// +// The context reaches a container's own planning (format.PlanWith), so a stop +// lands between two of the files inside it rather than after the last. +func planWithoutCrashing(ctx context.Context, desc format.Descriptor, r format.Request) (p format.Plan, err error) { defer func() { v := recover() if v == nil { @@ -88,7 +91,7 @@ func planWithoutCrashing(desc format.Descriptor, r format.Request) (p format.Pla } err = &GeneratorCrashError{Format: desc.ID, While: crashPlanning, Value: v} }() - return desc.Generator.Plan(r) + return format.PlanWith(ctx, desc.Generator, r) } // writeWithoutCrashing is the same around the write, so a crash costs one file diff --git a/internal/engine/drawsizes.go b/internal/engine/drawsizes.go index 96327656..1274ad07 100644 --- a/internal/engine/drawsizes.go +++ b/internal/engine/drawsizes.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "github.com/donislawdev/TestingFilesGenerator/internal/core" @@ -44,7 +45,7 @@ func firstWritable(desc format.Descriptor, r format.Request, from, limit int64) var first error for at := from; at <= limit; { r.Bytes = at - _, err := planWithoutCrashing(desc, r) + _, err := planWithoutCrashing(context.Background(), desc, r) if err == nil { return at, nil } @@ -128,7 +129,7 @@ func drawSizes(t *Target, desc format.Descriptor, targetSeed uint64) error { // Asked again at the low end so the format words its own refusal, with // the number the person actually wrote. first.Bytes = t.Range.Min - if _, err := planWithoutCrashing(desc, first); err != nil { + if _, err := planWithoutCrashing(context.Background(), desc, first); err != nil { return err } } diff --git a/internal/engine/plantarget.go b/internal/engine/plantarget.go index 70b664cc..0ae79a1d 100644 --- a/internal/engine/plantarget.go +++ b/internal/engine/plantarget.go @@ -35,7 +35,7 @@ func (pl *planning) files(ctx context.Context, t *Target, desc format.Descriptor for idx, size := range t.Sizes { fileSeed := core.FileSeed(targetSeed, idx) - p, err := planWithoutCrashing(desc, format.Request{ + p, err := planWithoutCrashing(ctx, desc, format.Request{ Bytes: size, SizeFromContents: t.SizeFromContents, Contains: t.Contains, diff --git a/internal/format/eml/attach.go b/internal/format/eml/attach.go index e561fa0f..036e0c27 100644 --- a/internal/format/eml/attach.go +++ b/internal/format/eml/attach.go @@ -1,6 +1,7 @@ package eml import ( + "context" "encoding/base64" "fmt" "io" @@ -64,12 +65,12 @@ func (c *child) raw() bool { return c.desc.MediaType == messageType } // per format across the whole message - the archives' rules, for their // reasons (zip/children.go). The order is the order the seeds were given in // before pictures and depth existed, so neither moves an attached file. -func planChildren(r format.Request, groups []format.Content, s settings) (children, pics []child, err error) { +func planChildren(ctx context.Context, r format.Request, groups []format.Content, s settings) (children, pics []child, err error) { total := 0 for _, g := range groups { total += g.Count } - p := planner{r: r, s: s, out: make([]child, 0, total+1), numbered: map[string]int{}} + p := planner{ctx: ctx, r: r, s: s, out: make([]child, 0, total+1), numbered: map[string]int{}} for _, g := range groups { desc, err := format.Get(g.Format) if err != nil { @@ -89,6 +90,7 @@ func planChildren(r format.Request, groups []format.Content, s settings) (childr // planner is planChildren's place in the message as it goes: how many files // came before, how many of each format, and how many bytes of them so far. type planner struct { + ctx context.Context r format.Request s settings out []child @@ -119,9 +121,9 @@ func (p *planner) attachment(c child) { // plan plans one file of a size with the seed of its place, and counts what it // adds to the message. func (p *planner) plan(desc format.Descriptor, size int64, place int) (child, error) { - cp, err := desc.Generator.Plan(p.request(desc, size, place)) + cp, err := format.PlanWith(p.ctx, desc.Generator, p.request(desc, size, place)) if err != nil { - return child{}, core.Refuse(core.Says("eml.TheAttachedFileCannotBe", "eml: the attached %s file cannot be made: %w", core.A("Format", desc.ID), core.A("Err", err))) + return child{}, format.StopOr(p.ctx, core.Refuse(core.Says("eml.TheAttachedFileCannotBe", "eml: the attached %s file cannot be made: %w", core.A("Format", desc.ID), core.A("Err", err)))) } if cp.Bytes > maxMessage { return child{}, tooLarge(cp.Bytes) @@ -161,7 +163,7 @@ func (p *planner) forward() error { r := p.request(desc, 0, len(p.out)) r.Properties[MessageDepth] = strconv.Itoa(p.s.depth - 1) r.SizeFromContents, r.Contains = true, []format.Content{} - cp, err := desc.Generator.Plan(r) + cp, err := format.PlanWith(p.ctx, desc.Generator, r) if err != nil { return err } diff --git a/internal/format/eml/eml.go b/internal/format/eml/eml.go index d96c697e..d748b66e 100644 --- a/internal/format/eml/eml.go +++ b/internal/format/eml/eml.go @@ -65,7 +65,13 @@ func minimumBytes() int64 { return newMessage(s, 0, nil, nil).size() } -func (generator) Plan(r format.Request) (format.Plan, error) { +func (g generator) Plan(r format.Request) (format.Plan, error) { + return g.PlanContext(context.Background(), r) +} + +// PlanContext is Plan, stoppable between one file the message carries and the +// next. +func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan, error) { s, err := parse(r.Properties) if err != nil { return format.Plan{}, err @@ -74,7 +80,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { if err != nil { return format.Plan{}, err } - children, pics, err := planChildren(r, groups, s) + children, pics, err := planChildren(ctx, r, groups, s) if err != nil { return format.Plan{}, err } diff --git a/internal/format/format.go b/internal/format/format.go index dfccf7fe..00fb379f 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -733,6 +733,45 @@ type Generator interface { Write(ctx context.Context, w io.Writer, p Plan) error } +// ContextPlanner is a Generator whose Plan can be stopped part of the way +// through, which is what a format that holds other files needs. +// +// A container plans every file inside it from inside its own Plan, and the +// engine asks for a stop only between one file and the next. Ten thousand +// small pngs in a page took 72 s to plan on 2026-10-14, the same in a mail +// message, a zip and a tar.gz, and a stopped run waited for all of it - under +// SIGTERM in CI long enough for SIGKILL to arrive (review of #178, +// docs/HTML-3-2026-10-14.md section 11). +type ContextPlanner interface { + PlanContext(ctx context.Context, r Request) (Plan, error) +} + +// PlanWith plans r with g, stoppable as far as g is: asked once before it +// starts, and between the files inside it when g is a ContextPlanner. +// +// A container plans its files through this too, so a container inside a +// container stops as soon as the outer one does. +func PlanWith(ctx context.Context, g Generator, r Request) (Plan, error) { + if err := ctx.Err(); err != nil { + return Plan{}, err + } + if c, ok := g.(ContextPlanner); ok { + return c.PlanContext(ctx, r) + } + return g.Plan(r) +} + +// StopOr is what a container returns when a file inside it could not be +// planned: the stop itself when the run was stopped, so it reaches the command +// line as a stop with its own exit code rather than as a file that cannot be +// made, and err otherwise. +func StopOr(ctx context.Context, err error) error { + if stop := ctx.Err(); stop != nil { + return stop + } + return err +} + // BelowMinimumError is refusing a size a format cannot deliver. // // It carries four things on purpose: which format, what its minimum is, why diff --git a/internal/format/htmlfile/html.go b/internal/format/htmlfile/html.go index cfb75473..c6efe840 100644 --- a/internal/format/htmlfile/html.go +++ b/internal/format/htmlfile/html.go @@ -78,13 +78,18 @@ type memo struct { pictureBytes int64 } -func (generator) Plan(r format.Request) (format.Plan, error) { +func (g generator) Plan(r format.Request) (format.Plan, error) { + return g.PlanContext(context.Background(), r) +} + +// PlanContext is Plan, stoppable between one picture and the next. +func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan, error) { s, err := settingsOf(r.Properties) if err != nil { return format.Plan{}, err } - pictures, pictureBytes, err := planPictures(r, s.pictures) + pictures, pictureBytes, err := planPictures(ctx, r, s.pictures) if err != nil { return format.Plan{}, err } diff --git a/internal/format/htmlfile/pictures.go b/internal/format/htmlfile/pictures.go index 2e16a48b..80d67482 100644 --- a/internal/format/htmlfile/pictures.go +++ b/internal/format/htmlfile/pictures.go @@ -64,7 +64,7 @@ func (p pictureLine) bytes() int64 { // asking for one more changes none of the others. The format is asked whether // it is a picture even when none are asked for: a value the setting cannot // take is a mistake either way, and eml refuses it the same way. -func planPictures(r format.Request, s picture.Settings) ([]pictureLine, int64, error) { +func planPictures(ctx context.Context, r format.Request, s picture.Settings) ([]pictureLine, int64, error) { desc, ok, err := picture.Of(s.Format) if err != nil { return nil, 0, err @@ -79,10 +79,10 @@ func planPictures(r format.Request, s picture.Settings) ([]pictureLine, int64, e out := make([]pictureLine, 0, s.Count) var sum int64 for i := 0; i < s.Count; i++ { - cp, err := desc.Generator.Plan(format.Request{Bytes: s.Size, Seed: core.FileSeed(r.Seed, i), Label: r.Label}) + cp, err := format.PlanWith(ctx, desc.Generator, format.Request{Bytes: s.Size, Seed: core.FileSeed(r.Seed, i), Label: r.Label}) if err != nil { - return nil, 0, core.Refuse(core.Says("html.ThePictureCannotBe", "html: the %s picture cannot be made: %w", - core.A("Format", desc.ID), core.A("Err", err))) + return nil, 0, format.StopOr(ctx, core.Refuse(core.Says("html.ThePictureCannotBe", "html: the %s picture cannot be made: %w", + core.A("Format", desc.ID), core.A("Err", err)))) } if cp.Bytes > maxPage { return nil, 0, pageTooLarge(cp.Bytes) diff --git a/internal/format/targz/targz.go b/internal/format/targz/targz.go index 3e372fa0..108e94a4 100644 --- a/internal/format/targz/targz.go +++ b/internal/format/targz/targz.go @@ -191,7 +191,12 @@ type memo struct { extraLen int64 } -func (generator) Plan(r format.Request) (format.Plan, error) { +func (g generator) Plan(r format.Request) (format.Plan, error) { + return g.PlanContext(context.Background(), r) +} + +// PlanContext is Plan, stoppable between one file inside and the next. +func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan, error) { // Before anything else, because every size below is worked out from this // and a framing this tool does not understand has to be a refusal rather // than an archive of the wrong length. See framing.go. @@ -220,7 +225,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } m := memo{seed: r.Seed, own: own, layout: layout, squeeze: squeeze} - if m.children, err = planChildren(r, groups, layout); err != nil { + if m.children, err = planChildren(ctx, r, groups, layout); err != nil { return format.Plan{}, err } @@ -253,7 +258,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { // Members are numbered across the whole archive rather than per group, so the // seed of a member does not move when a group above it changes count. That is // untouchable rule 2 applied one level down. -func planChildren(r format.Request, groups []format.Content, layout archive.Layout) ([]child, error) { +func planChildren(ctx context.Context, r format.Request, groups []format.Content, layout archive.Layout) ([]child, error) { // The directory chain depends only on the depth, so it is built once here // rather than once per entry. prefix := layout.Prefix() @@ -268,13 +273,13 @@ func planChildren(r format.Request, groups []format.Content, layout archive.Layo return nil, err } for i := 0; i < g.Count; i++ { - cp, err := desc.Generator.Plan(format.Request{ + cp, err := format.PlanWith(ctx, desc.Generator, format.Request{ Bytes: g.Bytes, Seed: core.FileSeed(r.Seed, index), Label: r.Label, }) if err != nil { - return nil, core.Defect(fmt.Errorf("targz: the %s file inside cannot be made: %w", g.Format, err)) + return nil, format.StopOr(ctx, core.Defect(fmt.Errorf("targz: the %s file inside cannot be made: %w", g.Format, err))) } numbered[g.Format]++ out = append(out, child{ diff --git a/internal/format/webp/picture.go b/internal/format/webp/picture.go index 3752cf89..c0c38f34 100644 --- a/internal/format/webp/picture.go +++ b/internal/format/webp/picture.go @@ -34,6 +34,9 @@ func writePicture(ctx context.Context, w io.Writer, m memo) (int64, error) { if y%64 == 0 && stopped == nil { stopped = interrupted(ctx) } + if stopped != nil { + return nil + } if y < bandH { copyBandRow(row, band, y, m.width) } else { diff --git a/internal/format/webp/vp8l.go b/internal/format/webp/vp8l.go index c424be21..a7697d92 100644 --- a/internal/format/webp/vp8l.go +++ b/internal/format/webp/vp8l.go @@ -170,7 +170,10 @@ func writeSingleSymbolCode(w *bitWriter, symbol uint32, eightBit bool) { } // writeStream emits the whole bitstream. row is called once per row and hands -// back width*3 bytes in red, green, blue order. +// back width*3 bytes in red, green, blue order, or nil to stop there - a run +// that was cancelled, whose file is thrown away, so the stream left short is +// never read (O279: until 2026-10-14 the rows went on to the last one after a +// stop, 5.2 s for a picture of 600 MB). func writeStream(w *bitWriter, width, height int, row func(y int) []byte) { w.write(0x2F, 8) w.write(uint32(width-1), 14) @@ -190,6 +193,9 @@ func writeStream(w *bitWriter, width, height int, row func(y int) []byte) { for y := 0; y < height; y++ { px := row(y) + if px == nil { + return + } for x := 0; x < width; x++ { // Green first. That is the order the format reads a pixel in, and // getting it wrong produces a picture that decodes cleanly with diff --git a/internal/format/zip/children.go b/internal/format/zip/children.go index d446f201..57ef0800 100644 --- a/internal/format/zip/children.go +++ b/internal/format/zip/children.go @@ -1,6 +1,7 @@ package zip import ( + "context" "fmt" "github.com/donislawdev/TestingFilesGenerator/internal/core" @@ -23,7 +24,7 @@ import ( // Members are numbered across the whole archive rather than per group, so the // seed of a member does not move when a group above it changes count. That is // untouchable rule 2 applied one level down. -func planChildren(r format.Request, groups []format.Content, layout archive.Layout) ([]child, error) { +func planChildren(ctx context.Context, r format.Request, groups []format.Content, layout archive.Layout) ([]child, error) { // The directory chain depends only on the depth, so it is built once here // rather than once per entry. prefix := layout.Prefix() @@ -48,13 +49,13 @@ func planChildren(r format.Request, groups []format.Content, layout archive.Layo } for i := 0; i < g.Count; i++ { childSeed := core.FileSeed(r.Seed, index) - cp, err := desc.Generator.Plan(format.Request{ + cp, err := format.PlanWith(ctx, desc.Generator, format.Request{ Bytes: g.Bytes, Seed: childSeed, Label: r.Label, }) if err != nil { - return nil, core.Refuse(core.Says("format.ZipTheFileInsideCannotBe", "zip: the %s file inside cannot be made: %w", core.A("Format", g.Format), core.A("Err", err))) + return nil, format.StopOr(ctx, core.Refuse(core.Says("format.ZipTheFileInsideCannotBe", "zip: the %s file inside cannot be made: %w", core.A("Format", g.Format), core.A("Err", err)))) } numbered[g.Format]++ out = append(out, child{ diff --git a/internal/format/zip/zip.go b/internal/format/zip/zip.go index 74ee9d6c..004943d3 100644 --- a/internal/format/zip/zip.go +++ b/internal/format/zip/zip.go @@ -128,7 +128,12 @@ type memo struct { target int64 } -func (generator) Plan(r format.Request) (format.Plan, error) { +func (g generator) Plan(r format.Request) (format.Plan, error) { + return g.PlanContext(context.Background(), r) +} + +// PlanContext is Plan, stoppable between one file inside and the next. +func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan, error) { groups, err := archive.Groups("zip", r) if err != nil { return format.Plan{}, err @@ -150,7 +155,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } m := memo{seed: r.Seed, lock: lock, layout: layout, squeeze: squeeze} - if m.children, err = planChildren(r, groups, layout); err != nil { + if m.children, err = planChildren(ctx, r, groups, layout); err != nil { return format.Plan{}, err } diff --git a/internal/guard/containerstop_test.go b/internal/guard/containerstop_test.go new file mode 100644 index 00000000..ea04fdfd --- /dev/null +++ b/internal/guard/containerstop_test.go @@ -0,0 +1,122 @@ +package guard + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// A format that holds other files stops planning them when the run is stopped. +// +// The engine asks for a stop between one file and the next, and a container +// plans every file inside it from inside its own Plan - ten thousand small +// pngs in a page took 72 s on 2026-10-14, and as long again in a mail message, +// a zip and a tar.gz, with Ctrl+C waiting for the last of them (review of +// #178, docs/HTML-3-2026-10-14.md section 11). + +// countdown is a context that is live for a number of questions and stopped +// from then on, so a stop lands part of the way through planning without a +// clock: a container that asks between its files sees it, and one that asks +// only before it starts does not. +type countdown struct { + context.Context + mu sync.Mutex + left int + done chan struct{} +} + +func newCountdown(left int) *countdown { + return &countdown{Context: context.Background(), left: left, done: make(chan struct{})} +} + +func (c *countdown) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + if c.left > 0 { + c.left-- + return nil + } + select { + case <-c.done: + default: + close(c.done) + } + return context.Canceled +} + +func (c *countdown) Done() <-chan struct{} { return c.done } + +// containerRequests are the four formats that plan files inside them, each +// asked for forty, which a live context plans in well under a second. +func containerRequests() map[string]format.Request { + forty := []format.Content{{Format: "txt", Count: 40, Bytes: 1024}} + pictures := map[string]string{"inline": "40", "inline_size": "1kb"} + return map[string]format.Request{ + "html": {Bytes: 1 << 20, Seed: 7741, Label: true, Properties: pictures}, + "eml": {Bytes: 1 << 20, Seed: 7741, Label: true, Properties: pictures}, + "zip": {Bytes: 1 << 20, Seed: 7741, Label: true, Contains: forty}, + "targz": {Bytes: 1 << 20, Seed: 7741, Label: true, Contains: forty}, + } +} + +// TestAContainerStopsBetweenTheFilesItPlans asks each container twice. With a +// live context it has to plan - otherwise the stop below could be a refusal of +// something else and pass for the thing guarded. With a context that stops +// after three questions it has to end with the stop itself, and NOT as a +// refusal: a window marks the box a refusal is about, and a stop is the answer +// to what was asked for rather than a fault in any box. +func TestAContainerStopsBetweenTheFilesItPlans(t *testing.T) { + for id, r := range containerRequests() { + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + if _, ok := d.Generator.(format.ContextPlanner); !ok { + t.Errorf("%s plans files inside it and cannot be stopped while it does", id) + continue + } + if _, err := format.PlanWith(context.Background(), d.Generator, r); err != nil { + t.Errorf("%s: forty files inside are not planned with a live context, so the stop is not asked: %v", id, err) + continue + } + _, err = format.PlanWith(newCountdown(3), d.Generator, r) + if !errors.Is(err, context.Canceled) { + t.Errorf("%s: stopped after three questions and planning ended with %v, not the stop", id, err) + } + var refusal *core.Refusal + if errors.As(err, &refusal) { + t.Errorf("%s: a stop came back as a refusal, %q, so a window would mark a box for it", id, err) + } + } +} + +// TestTheEngineHandsTheStopToAContainerItIsPlanning is the other half: the +// containers listen, and the engine has to be the one talking to them. A page +// of ten thousand small pictures takes over a minute to plan, so a run stopped +// a moment in has to end long before that, with the stop. +func TestTheEngineHandsTheStopToAContainerItIsPlanning(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + time.Sleep(200 * time.Millisecond) + cancel() + }() + start := time.Now() + _, err := engine.PlanContext(ctx, []engine.Target{{ + ID: "g", Format: "html", Sizes: engine.Uniform(1, 1<<30), Label: true, + Properties: map[string]string{"inline": "10000", "inline_size": "8kb"}, + }}, engine.Options{OutDir: t.TempDir(), Seed: 7741}) + took := time.Since(start) + if !errors.Is(err, context.Canceled) { + t.Errorf("planning a page of ten thousand pictures, stopped after 200 ms, ended with %v after %v, not the stop", err, took) + } + if took > 20*time.Second { + t.Errorf("planning stopped after 200 ms took %v to notice", took) + } +} diff --git a/internal/guard/generatorcrash_test.go b/internal/guard/generatorcrash_test.go index 9e5b2fe7..71e7f096 100644 --- a/internal/guard/generatorcrash_test.go +++ b/internal/guard/generatorcrash_test.go @@ -117,7 +117,11 @@ func TestAGeneratorThatCrashesWhileWritingCostsOneFile(t *testing.T) { // borrowed encoder. func TestEveryCallIntoAGeneratorGoesThroughTheCrashWrapper(t *testing.T) { const wrapper = "crash.go" - calls := []string{"Generator.Plan(", "Generator.Write("} + // format.PlanWith is how the wrapper plans since 2026-10-14, so a + // container can be stopped between the files inside it - and it calls the + // generator, so anywhere else in the engine it is a way round the wrapper. + calls := []string{"format.PlanWith(", "Generator.Write("} + direct := "Generator.Plan(" dir := filepath.Join(repoRoot(t), "internal", "engine") entries, err := os.ReadDir(dir) @@ -134,7 +138,7 @@ func TestEveryCallIntoAGeneratorGoesThroughTheCrashWrapper(t *testing.T) { t.Fatalf("reading %s: %v", e.Name(), err) } read++ - for _, call := range calls { + for _, call := range append(calls, direct) { if strings.Contains(string(body), call) { t.Errorf("%s calls %s itself, so a panic in there ends the process", e.Name(), strings.TrimSuffix(call, "(")) diff --git a/internal/guard/htmlpictures_test.go b/internal/guard/htmlpictures_test.go index ae7646b7..7e06ae28 100644 --- a/internal/guard/htmlpictures_test.go +++ b/internal/guard/htmlpictures_test.go @@ -6,6 +6,8 @@ import ( "encoding/base64" "errors" "fmt" + "os" + "path/filepath" "regexp" "strings" "testing" @@ -13,6 +15,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" pic "github.com/donislawdev/TestingFilesGenerator/internal/format/picture" + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" ) // The pictures a page carries inside itself, from 2026-10-14 @@ -152,3 +155,32 @@ func TestEveryFormatWithPicturesDeclaresThemAlike(t *testing.T) { t.Errorf("%d formats carry pictures, and eml and html both do", carriers) } } + +// TestTheStructuralCheckGivesAVerdictOnABrokenPicture holds the review of #178: +// a picture the size it should be whose bytes stop a format's own check part +// way - a PNG whose last chunk runs to the very end, so its checksum is not +// there to unpack - is a verdict about the picture, not a traceback that +// reads as the checker saying nothing useful. +func TestTheStructuralCheckGivesAVerdictOnABrokenPicture(t *testing.T) { + props := map[string]string{"inline": "1", "inline_size": "4kb"} + body := writeEncoded(t, "html", 65536, props) + m := htmlPictureLine.FindSubmatchIndex(body) + if m == nil { + t.Fatal("the page carries no picture line, so there is nothing to break") + } + broken := append([]byte("\x89PNG\r\n\x1a\n"), 0, 0, 0x0f, 0xf0) + broken = append(broken, "IHDR"...) + broken = append(broken, make([]byte, 4096-len(broken))...) + page := append(append(append([]byte{}, body[:m[8]]...), base64.StdEncoding.EncodeToString(broken)...), body[m[9]:]...) + path := filepath.Join(t.TempDir(), "broken.html") + if err := os.WriteFile(path, page, 0o600); err != nil { + t.Fatal(err) + } + res := oracle.Strict("html", path, htmlSettingWords(props)...) + if !res.Available { + t.Skip("the structural check needs python") + } + if res.Err == nil || !strings.HasPrefix(res.Err.Error(), "picture 1 is not a good png file") { + t.Errorf("a page with a broken picture was answered with %v, not a verdict on the picture", res.Err) + } +} diff --git a/internal/guard/picturestop_test.go b/internal/guard/picturestop_test.go new file mode 100644 index 00000000..4696c862 --- /dev/null +++ b/internal/guard/picturestop_test.go @@ -0,0 +1,69 @@ +package guard + +import ( + "context" + "errors" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + pic "github.com/donislawdev/TestingFilesGenerator/internal/format/picture" +) + +// A picture format stops writing soon after the run it belongs to is stopped. +// +// Ctrl+C reaches a generator only through its context, and a generator that +// notices it and writes on regardless keeps a stopped run going: webp did, +// coding a picture to its last row after the stop, 5.2 s for 600 MB (O279, +// measured 2026-10-14 through a page carrying the picture). A picture inside a +// page or a mail message is written by the same generator, so this holds them +// too. +// +// Counted in bytes rather than timed, so a slow machine cannot turn it red and +// a fast one cannot hide the defect: the writer below stops the run once the +// first MiB has reached it, and what comes after has to be a small part of the +// file rather than all of it. + +// stoppingWriter cancels a run once a number of bytes have reached it, and +// keeps counting what still arrives. +type stoppingWriter struct { + cancel context.CancelFunc + after int64 + n int64 +} + +func (s *stoppingWriter) Write(p []byte) (int, error) { + s.n += int64(len(p)) + if s.n >= s.after { + s.cancel() + } + return len(p), nil +} + +func TestEveryPictureFormatStopsSoonAfterARunIsStopped(t *testing.T) { + const size, stopAt = int64(64 << 20), int64(1 << 20) + formats := pic.Formats() + if len(formats) == 0 { + t.Fatal("this build has no picture format, so nothing below is asked") + } + for _, id := range formats { + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + p, err := d.Generator.Plan(format.Request{Bytes: size, Seed: 7741, Label: true}) + if err != nil { + t.Errorf("%s: %d B cannot be planned, so its stop is not asked: %v", id, size, err) + continue + } + ctx, cancel := context.WithCancel(context.Background()) + w := &stoppingWriter{cancel: cancel, after: stopAt} + err = d.Generator.Write(ctx, w, p) + cancel() + if !errors.Is(err, context.Canceled) { + t.Errorf("%s: stopped after 1 MiB of %d B and the write ended with %v, not the stop", id, size, err) + } + if w.n > size/4 { + t.Errorf("%s: stopped after 1 MiB and wrote %d B of %d B before it noticed", id, w.n, size) + } + } +} diff --git a/internal/oracle/strict.py b/internal/oracle/strict.py index fe270581..7d6a7b7f 100644 --- a/internal/oracle/strict.py +++ b/internal/oracle/strict.py @@ -819,7 +819,12 @@ def html_pictures(text, settings): def html_picture_verdict(kind, n, raw): """One picture judged by its own format's check. Those end the process with - their verdict, so each is caught here and only a failure is passed on.""" + their verdict, so each is caught here and only a failure is passed on. + + A check can also stop on bytes it did not expect - check_png unpacks a + checksum that is not there when a chunk runs to the very end - and that is + a verdict about the picture too, so it is given as one rather than as a + traceback (review of #178).""" import contextlib import io @@ -831,6 +836,8 @@ def html_picture_verdict(kind, n, raw): if exc.code not in (0, None): fail(f"picture {n} is not a good {kind} file: {said.getvalue().strip()}") return + except Exception as exc: + fail(f"picture {n} is not a good {kind} file: {type(exc).__name__}: {str(exc)[:120]}") fail(f"the {kind} check gave no verdict on picture {n}")