html: pictures inside the page, with the settings eml pictures have

CHANGELOG.md

@@ -157,6 +157,17 @@ because it turns other people's test suites red.
   describes, which Go's `net/http` follows - so a filter that recognises
   HTML by its first bytes can let it through. Files made with the earlier
   settings keep their bytes.
+- **HTML: pictures inside the page.** `inline`, `inline_format` and
+  `inline_size` - the same settings, with the same meaning, as the pictures
+  of an `eml` message - write real pictures of any picture format into the
+  page as `data:` URLs, each on a line of its own above the blocks, so
+  opening the page fetches nothing. Each picture is the very file its format
+  writes on its own. The Nu HTML Checker finds nothing wrong with a page of
+  any of the nine formats, Chromium shows every format but `tiff`, and
+  DOMPurify keeps them all. lxml, and BeautifulSoup on top of it, leave out
+  a picture past about 7.1 MB unless told to read huge trees - on Linux lxml
+  took over 20 seconds over such a page, and with a 15 MB picture it lost the
+  rest of the page as well. Files made without pictures keep their bytes.
 - **A length of time as a setting.** `duration`, `change_interval` and `keyframe_interval` take
   `10s`, `1m30s`, `1h`, `500ms` or `59.9s`, to the millisecond. A bare number
   is refused rather than read as seconds, and so is a length that does not
@@ -266,6 +277,18 @@ because it turns other people's test suites red.
 
 ### Fixed
 
+- **Ctrl+C waited for a file that holds thousands of files to be planned.**
+  A zip, a tar.gz, a mail message or a page with ten thousand small pictures
+  inside took over a minute to plan, and a run stopped in that minute went
+  on until the end of it - in CI long enough for the grace period to run
+  out. Planning now stops between one file inside and the next, and says
+  it stopped rather than that a file could not be made.
+
+- **A stopped run went on writing a WebP picture to its last row.** A
+  600 MB picture took five more seconds to stop. It now stops within a few
+  rows, as the other picture formats do. Files that are not stopped keep
+  their bytes.
+
 - **Choosing another tool on the Tools tab left the boxes in the wrong
   place.** The section kept the height of the tool chosen before, so a
   setting could stand under it with its list hidden, or an empty band was

README.md

@@ -621,7 +621,7 @@ recipe. `tfg formats <id>` prints the allowed range or list for each:
 | `geojson` | `geometry`, `formatting`, `precision`, `altitude`, `vertices`, `holes`, `winding`, `antimeridian`, `unlocated`, `ids`, `bbox` |
 | `eml` | `body`, `text_encoding`, `charset`, `headers`, `recipients`, `line_endings`, `attachments`, `attachment_format`, `attachment_size`, `filename_style`, `message_depth`, `inline`, `inline_format`, `inline_size` |
 | `svg` | `width`, `height` |
-| `html` | `structure`, `nesting`, `characters`, `elements`, `bom` |
+| `html` | `structure`, `nesting`, `characters`, `elements`, `bom`, `inline`, `inline_format`, `inline_size` |
 | `yaml`, `toml` | none in this build - the document is a fixed shape, and a size is the only thing to ask for |
 
 ```

internal/engine/crash.go

@@ -80,15 +80,18 @@ const (
 // container planning its children calls them from inside its own Plan, so a
 // child crashing is caught here too - and named as the container, because the
 // container is what this was asked to plan.
-func planWithoutCrashing(desc format.Descriptor, r format.Request) (p format.Plan, err error) {
+//
+// The context reaches a container's own planning (format.PlanWith), so a stop
+// lands between two of the files inside it rather than after the last.
+func planWithoutCrashing(ctx context.Context, desc format.Descriptor, r format.Request) (p format.Plan, err error) {
 	defer func() {
 		v := recover()
 		if v == nil {
 			return
 		}
 		err = &GeneratorCrashError{Format: desc.ID, While: crashPlanning, Value: v}
 	}()
-	return desc.Generator.Plan(r)
+	return format.PlanWith(ctx, desc.Generator, r)
 }
 
 // writeWithoutCrashing is the same around the write, so a crash costs one file

internal/engine/drawsizes.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"context"
 	"errors"
 
 	"github.com/donislawdev/TestingFilesGenerator/internal/core"
@@ -44,7 +45,7 @@ func firstWritable(desc format.Descriptor, r format.Request, from, limit int64)
 	var first error
 	for at := from; at <= limit; {
 		r.Bytes = at
-		_, err := planWithoutCrashing(desc, r)
+		_, err := planWithoutCrashing(context.Background(), desc, r)
 		if err == nil {
 			return at, nil
 		}
@@ -128,7 +129,7 @@ func drawSizes(t *Target, desc format.Descriptor, targetSeed uint64) error {
 		// Asked again at the low end so the format words its own refusal, with
 		// the number the person actually wrote.
 		first.Bytes = t.Range.Min
-		if _, err := planWithoutCrashing(desc, first); err != nil {
+		if _, err := planWithoutCrashing(context.Background(), desc, first); err != nil {
 			return err
 		}
 	}

internal/engine/plantarget.go

@@ -35,7 +35,7 @@ func (pl *planning) files(ctx context.Context, t *Target, desc format.Descriptor
 	for idx, size := range t.Sizes {
 		fileSeed := core.FileSeed(targetSeed, idx)
 
-		p, err := planWithoutCrashing(desc, format.Request{
+		p, err := planWithoutCrashing(ctx, desc, format.Request{
 			Bytes:            size,
 			SizeFromContents: t.SizeFromContents,
 			Contains:         t.Contains,

internal/format/eml/attach.go

@@ -1,6 +1,7 @@
 package eml
 
 import (
+	"context"
 	"encoding/base64"
 	"fmt"
 	"io"
@@ -64,12 +65,12 @@ func (c *child) raw() bool { return c.desc.MediaType == messageType }
 // per format across the whole message - the archives' rules, for their
 // reasons (zip/children.go). The order is the order the seeds were given in
 // before pictures and depth existed, so neither moves an attached file.
-func planChildren(r format.Request, groups []format.Content, s settings) (children, pics []child, err error) {
+func planChildren(ctx context.Context, r format.Request, groups []format.Content, s settings) (children, pics []child, err error) {
 	total := 0
 	for _, g := range groups {
 		total += g.Count
 	}
-	p := planner{r: r, s: s, out: make([]child, 0, total+1), numbered: map[string]int{}}
+	p := planner{ctx: ctx, r: r, s: s, out: make([]child, 0, total+1), numbered: map[string]int{}}
 	for _, g := range groups {
 		desc, err := format.Get(g.Format)
 		if err != nil {
@@ -89,6 +90,7 @@ func planChildren(r format.Request, groups []format.Content, s settings) (childr
 // planner is planChildren's place in the message as it goes: how many files
 // came before, how many of each format, and how many bytes of them so far.
 type planner struct {
+	ctx      context.Context
 	r        format.Request
 	s        settings
 	out      []child
@@ -119,9 +121,9 @@ func (p *planner) attachment(c child) {
 // plan plans one file of a size with the seed of its place, and counts what it
 // adds to the message.
 func (p *planner) plan(desc format.Descriptor, size int64, place int) (child, error) {
-	cp, err := desc.Generator.Plan(p.request(desc, size, place))
+	cp, err := format.PlanWith(p.ctx, desc.Generator, p.request(desc, size, place))
 	if err != nil {
-		return child{}, core.Refuse(core.Says("eml.TheAttachedFileCannotBe", "eml: the attached %s file cannot be made: %w", core.A("Format", desc.ID), core.A("Err", err)))
+		return child{}, format.StopOr(p.ctx, core.Refuse(core.Says("eml.TheAttachedFileCannotBe", "eml: the attached %s file cannot be made: %w", core.A("Format", desc.ID), core.A("Err", err))))
 	}
 	if cp.Bytes > maxMessage {
 		return child{}, tooLarge(cp.Bytes)
@@ -161,7 +163,7 @@ func (p *planner) forward() error {
 	r := p.request(desc, 0, len(p.out))
 	r.Properties[MessageDepth] = strconv.Itoa(p.s.depth - 1)
 	r.SizeFromContents, r.Contains = true, []format.Content{}
-	cp, err := desc.Generator.Plan(r)
+	cp, err := format.PlanWith(p.ctx, desc.Generator, r)
 	if err != nil {
 		return err
 	}

internal/format/eml/eml.go

@@ -18,6 +18,7 @@ import (
 	"github.com/donislawdev/TestingFilesGenerator/internal/core"
 	"github.com/donislawdev/TestingFilesGenerator/internal/format"
 	"github.com/donislawdev/TestingFilesGenerator/internal/format/archive"
+	"github.com/donislawdev/TestingFilesGenerator/internal/format/picture"
 )
 
 const generatorVersion = "1"
@@ -64,7 +65,13 @@ func minimumBytes() int64 {
 	return newMessage(s, 0, nil, nil).size()
 }
 
-func (generator) Plan(r format.Request) (format.Plan, error) {
+func (g generator) Plan(r format.Request) (format.Plan, error) {
+	return g.PlanContext(context.Background(), r)
+}
+
+// PlanContext is Plan, stoppable between one file the message carries and the
+// next.
+func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan, error) {
 	s, err := parse(r.Properties)
 	if err != nil {
 		return format.Plan{}, err
@@ -73,7 +80,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
 	if err != nil {
 		return format.Plan{}, err
 	}
-	children, pics, err := planChildren(r, groups, s)
+	children, pics, err := planChildren(ctx, r, groups, s)
 	if err != nil {
 		return format.Plan{}, err
 	}
@@ -242,19 +249,15 @@ func forwarded(m *message) int {
 	return 0
 }
 
-// describePictures adds the pictures to what reaches the manifest - always
-// their settings, so a test reads the same keys whether there are any or not.
+// describePictures adds the pictures to what reaches the manifest, in the
+// words html uses for its own.
 func describePictures(props map[string]any, m *message) {
 	var raw, encoded int64
 	for _, c := range m.pictures {
 		raw += c.plan.Bytes
 		encoded += c.encoded
 	}
-	props[Inline] = len(m.pictures)
-	props[InlineFormat] = m.s.pictures.format
-	props[InlineSize] = m.s.pictures.size
-	props["inline_bytes"] = raw
-	props["inline_encoded_bytes"] = encoded
+	picture.Describe(props, m.s.pictures, len(m.pictures), raw, encoded)
 }
 
 func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error {

internal/format/eml/inline.go

@@ -2,12 +2,11 @@ package eml
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 
 	"github.com/donislawdev/TestingFilesGenerator/internal/core"
 	"github.com/donislawdev/TestingFilesGenerator/internal/format"
-	"github.com/donislawdev/TestingFilesGenerator/internal/format/archive"
+	"github.com/donislawdev/TestingFilesGenerator/internal/format/picture"
 )
 
 // Pictures in the text: real files of a picture format, carried beside the
@@ -21,97 +20,40 @@ import (
 // lists them when the HTML is alone, MimeKit never does. A system that limits
 // or counts attachments meets the same file three ways.
 
-// Setting names, public under rule 10.
+// Setting names, public under rule 10. The declaration and its reading are
+// shared with html, which carries pictures inside the page (package picture).
 const (
-	Inline       = "inline"
-	InlineFormat = "inline_format"
-	InlineSize   = "inline_size"
+	Inline       = picture.Count
+	InlineFormat = picture.Format
+	InlineSize   = picture.Size
 )
 
 // picturesGroup is the block the picture settings sit in, on both surfaces.
 const picturesGroup = "Pictures in the text"
 
-// defaultPictureFormat is what a picture is when nothing says - the format a
-// mail program puts a pasted picture in.
-const defaultPictureFormat = "png"
-
-// pictures is what the inline settings ask for.
-type pictures struct {
-	count  int
-	format string
-	size   int64
-}
-
 // pictureProperties are the settings of the pictures the text shows.
 func pictureProperties() []format.Property {
-	return []format.Property{
-		{
-			Name: Inline, Kind: format.PropertyInt,
-			Min: 0, Max: archive.MaxMembers,
-			Default: "0",
-			Detail: "How many pictures the text shows. With html and alternative each one sits beside the HTML and the HTML points at it through cid:. " +
-				"With plain there is no HTML to point from, so they travel after the text as pictures to show inline. " +
-				"mailparser lists them among the attachments, and so does Python's email package when the HTML is alone. MimeKit never does.",
-			Group: picturesGroup,
-		},
-		{
-			Name: InlineFormat, Kind: format.PropertyText,
-			Shape:   "the id of a picture format, as tfg formats lists them",
-			Default: defaultPictureFormat,
-			Detail:  "The format of the pictures. Any format whose files are pictures, such as png, jpg or gif.",
-			Group:   picturesGroup,
-		},
-		{
-			Name: InlineSize, Kind: format.PropertySize,
-			Default: archive.DefaultSizeText,
-			Detail:  "How big each picture is. Like an attached file it travels in base64, about a third larger.",
-			Group:   picturesGroup,
-		},
-	}
-}
-
-// readPictures is the three settings as numbers. Whether the format is a
-// picture is asked in Plan, where the registry is complete - this also runs
-// while the formats are still registering, to work out the minimum.
-func readPictures(values map[string]string) (pictures, error) {
-	n, err := strconv.Atoi(values[Inline])
-	if err != nil {
-		return pictures{}, core.Defect(err)
-	}
-	size, err := core.ParseSize(values[InlineSize])
-	if err != nil {
-		return pictures{}, core.Defect(err)
-	}
-	return pictures{count: n, format: values[InlineFormat], size: size}, nil
+	return picture.Properties(picturesGroup, picture.Details{
+		Count: "How many pictures the text shows. With html and alternative each one sits beside the HTML and the HTML points at it through cid:. " +
+			"With plain there is no HTML to point from, so they travel after the text as pictures to show inline. " +
+			"mailparser lists them among the attachments, and so does Python's email package when the HTML is alone. MimeKit never does.",
+		Format: "The format of the pictures. Any format whose files are pictures, such as png, jpg or gif.",
+		Size:   "How big each picture is. Like an attached file it travels in base64, about a third larger.",
+	})
 }
 
 // pictureFormat is the descriptor of the format the pictures are, or the
 // refusal when it is not a picture.
 func pictureFormat(id string) (format.Descriptor, error) {
-	desc, err := format.Get(id)
-	if err != nil {
-		return format.Descriptor{}, err
-	}
-	if strings.HasPrefix(desc.MediaType, "image/") {
-		return desc, nil
+	desc, ok, err := picture.Of(id)
+	if err != nil || ok {
+		return desc, err
 	}
 	return format.Descriptor{}, &format.PropertyValueError{
 		Format: "eml", Key: InlineFormat, Value: id,
 		Reason: core.Says("eml.APictureInTheTextIs", "a picture in the text has to be a file a mail program can show as a picture"),
-		Remedy: core.Says("eml.UseOneOfThePictureFormats", "Use one of: %s.", core.A("Formats", strings.Join(pictureFormats(), ", "))),
-	}
-}
-
-// pictureFormats are the formats of this build whose files are pictures, from
-// the media type each declares rather than from a list kept here.
-func pictureFormats() []string {
-	var out []string
-	for _, d := range format.All() {
-		if strings.HasPrefix(d.MediaType, "image/") {
-			out = append(out, d.ID)
-		}
+		Remedy: core.Says("eml.UseOneOfThePictureFormats", "Use one of: %s.", core.A("Formats", strings.Join(picture.Formats(), ", "))),
 	}
-	return out
 }
 
 // addPictures plans the pictures, after the attached files, with seeds that
@@ -121,14 +63,14 @@ func pictureFormats() []string {
 // value the setting cannot take is a mistake either way, and accepting it in
 // silence would hand back a message the recipe did not describe.
 func (p *planner) addPictures(s settings) ([]child, error) {
-	desc, err := pictureFormat(s.pictures.format)
-	if err != nil || s.pictures.count == 0 {
+	desc, err := pictureFormat(s.pictures.Format)
+	if err != nil || s.pictures.Count == 0 {
 		return nil, err
 	}
-	out := make([]child, 0, s.pictures.count)
+	out := make([]child, 0, s.pictures.Count)
 	first := len(p.out)
-	for i := 0; i < s.pictures.count; i++ {
-		c, err := p.plan(desc, s.pictures.size, first+i)
+	for i := 0; i < s.pictures.Count; i++ {
+		c, err := p.plan(desc, s.pictures.Size, first+i)
 		if err != nil {
 			return nil, err
 		}

internal/format/eml/settings.go

@@ -6,6 +6,7 @@ import (
 	"github.com/donislawdev/TestingFilesGenerator/internal/core"
 	"github.com/donislawdev/TestingFilesGenerator/internal/format"
 	"github.com/donislawdev/TestingFilesGenerator/internal/format/archive"
+	"github.com/donislawdev/TestingFilesGenerator/internal/format/picture"
 )
 
 // Setting names. Public names under rule 10, so each is spelled once.
@@ -110,7 +111,7 @@ type settings struct {
 	// messages deep the message goes, itself counted.
 	recipients int
 	depth      int
-	pictures   pictures
+	pictures   picture.Settings
 }
 
 // international says whether the text carries the line in Polish (and in
@@ -174,7 +175,7 @@ func parse(props map[string]string) (settings, error) {
 	if s.depth, err = strconv.Atoi(values[MessageDepth]); err != nil {
 		return settings{}, core.Defect(err)
 	}
-	s.pictures, err = readPictures(values)
+	s.pictures, err = picture.Parse(values)
 	return s, err
 }