From 20958aa9db5c3d288028f30c9c4a49e325dbdf1b Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:11:48 -0400 Subject: [PATCH 01/98] xsd: clip document strings in validation messages fail clips every string and error argument and the formatted message to 1 KB; attrName clips before fail sees the name; the pattern-facet and list-item errors quote their value through truncate. 200 foreign attributes in a 1 MB namespace: 105.7 MB live / 422.8 MB allocated -> 0.13 MB / 0.83 MB. --- xsd/facet.go | 2 +- xsd/message_clip_test.go | 51 ++++++++++++++++++++++++++++++++++++++++ xsd/validate.go | 24 ++++++++++++++++--- xsd/validate_attr.go | 4 +++- xsd/validate_simple.go | 2 +- 5 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 xsd/message_clip_test.go diff --git a/xsd/facet.go b/xsd/facet.go index 665cb3ac..1ec4f0e8 100644 --- a/xsd/facet.go +++ b/xsd/facet.go @@ -566,7 +566,7 @@ func checkPatterns(steps []facetStep, lexical string) error { } if !ok { return facetError(st.typ, FacetPattern, - "%q does not match %s", lexical, describePatterns(st.facets.Patterns)) + "%q does not match %s", truncate(lexical), describePatterns(st.facets.Patterns)) } } return nil diff --git a/xsd/message_clip_test.go b/xsd/message_clip_test.go new file mode 100644 index 00000000..64819edf --- /dev/null +++ b/xsd/message_clip_test.go @@ -0,0 +1,51 @@ +package xsd + +import ( + "errors" + "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// A name or value the document supplies is clipped before it reaches a +// ValidationError. 200 foreign attributes in a 1 MB namespace kept 105 MB in +// the errors and allocated 423 MB building them; clipped, under 1 MB of each. +// The allocation bound catches attrName and fail independently: without the +// clip in attrName it is about 212 MB, without the one in fail about 105 MB. +func TestValidationMessagesClipDocumentStrings(t *testing.T) { + s := mustParseSchema(t, ` + + `) + uri := strings.Repeat("u", 1<<20) + var b strings.Builder + b.WriteString(``) + tree, err := xdm.ParseString(b.String(), xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var m0, m1 runtime.MemStats + runtime.ReadMemStats(&m0) + err = s.Validate(tree.Root, ValidateOptions{}) + runtime.ReadMemStats(&m1) + var ve *ValidationErrors + if !errors.As(err, &ve) || len(ve.Errors) == 0 { + t.Fatalf("expected ValidationErrors, got %v", err) + } + for _, e := range ve.Errors { + if len(e.Message) > maxMessage+len("...") { + t.Fatalf("message is %d bytes, want at most %d", len(e.Message), maxMessage+3) + } + if !strings.Contains(e.Message, "{uuuu") || !strings.Contains(e.Message, "...") { + t.Fatalf("message lost its prefix or its clip marker: %.200q", e.Message) + } + } + if got := m1.TotalAlloc - m0.TotalAlloc; got > 16<<20 { + t.Errorf("validation allocated %d MB, want under 16 MB", got>>20) + } +} diff --git a/xsd/validate.go b/xsd/validate.go index e74f2e57..33246d9d 100644 --- a/xsd/validate.go +++ b/xsd/validate.go @@ -535,9 +535,19 @@ func (v *validator) fail(n *xdm.Node, code, format string, args ...any) { v.stopped = true return } + // Arguments are clipped before formatting so the full string is never + // copied, and the message after, for arguments that are neither. + for i, a := range args { + switch x := a.(type) { + case string: + args[i] = clip(x, maxMessage) + case error: + args[i] = clip(x.Error(), maxMessage) + } + } e := &ValidationError{ Code: code, - Message: fmt.Sprintf(format, args...), + Message: clip(fmt.Sprintf(format, args...), maxMessage), Path: v.pathString(), } if t := v.twins[n]; t != nil { @@ -1131,8 +1141,16 @@ func nonSpaceTextSeen(el *xdm.Node, seen *string) string { return "" } -func truncate(s string) string { - const max = 40 +func truncate(s string) string { return clip(s, 40) } + +// maxMessage bounds what one ValidationError keeps of a name or value the +// document supplied. A 15 MB namespace URI quoted in each of 100 errors +// retained 1.57 GB; real names and values are far shorter than the bound. +const maxMessage = 1024 + +// clip shortens s to max bytes, marking the cut with "...". The concatenation +// copies, so the result never pins the original string. +func clip(s string, max int) string { if len(s) <= max { return s } diff --git a/xsd/validate_attr.go b/xsd/validate_attr.go index 6b730a86..7d225f06 100644 --- a/xsd/validate_attr.go +++ b/xsd/validate_attr.go @@ -341,7 +341,9 @@ func attrName(q xdm.QName) string { if q.URI == "" { return q.Local } - return "{" + q.URI + "}" + q.Local + // Clipped here as well as in fail, because the name is built before + // fail can see it, and before fail knows whether MaxErrors drops it. + return "{" + clip(q.URI, maxMessage) + "}" + clip(q.Local, maxMessage) } // validateAttribute checks one attribute against its declaration. diff --git a/xsd/validate_simple.go b/xsd/validate_simple.go index 34a685e9..1d0aab04 100644 --- a/xsd/validate_simple.go +++ b/xsd/validate_simple.go @@ -373,7 +373,7 @@ func validateListValueIn(lexical string, t *SimpleType, at *instance) (string, e if t.ItemType != nil { for _, item := range items { if _, err := validateSimpleValueIn(item, t.ItemType, Version10, at); err != nil { - return "", fmt.Errorf("list item %q: %w", item, err) + return "", fmt.Errorf("list item %q: %w", truncate(item), err) } } } From 18d374d3e1ab79a69b1a1bb8fd06248e414f25cd Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:11:56 -0400 Subject: [PATCH 02/98] xdm: copy namespace URIs out of the tokeniser arena A namespace URI was an arena string, and the tree keeps it in every name it resolves, so each parsed document kept a 32 KB arena block alive. Retained heap for the DocBook corpus (591 trees): 31.6 MB -> 12.3 MB. --- xdm/parse.go | 2 +- xdm/retain_test.go | 54 ++++++++++++++++++++++++++++++++++++ xdm/wellformed.go | 5 +++- xdm/wellformed_scope_test.go | 4 +-- 4 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 xdm/retain_test.go diff --git a/xdm/parse.go b/xdm/parse.go index e3f3b835..83162da6 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -795,7 +795,7 @@ func buildElement(t xml.StartElement, parent *Node, ns *nsScope, names *nameCach if a.Name.Space == "xml" { switch a.Name.Local { case "base": - el.SetBaseURI(resolveBase(el.BaseURI(), a.Value)) + el.SetBaseURI(resolveBase(el.BaseURI(), strings.Clone(a.Value))) // out of the arena, as nsScope.declare case "id": // xml:id is an ID by definition, and the xml:id // Recommendation requires its value to be normalised as one, diff --git a/xdm/retain_test.go b/xdm/retain_test.go new file mode 100644 index 00000000..d33e6655 --- /dev/null +++ b/xdm/retain_test.go @@ -0,0 +1,54 @@ +package xdm + +import ( + "runtime" + "strings" + "testing" +) + +// retainedPerTree is the heap still live, per tree, with n parses of doc +// kept, after a collection. +func retainedPerTree(t *testing.T, doc string, opts ParseOptions, n int) int64 { + t.Helper() + trees := make([]*Tree, n) + var m0, m1 runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&m0) + for i := range trees { + tr, err := ParseString(doc, opts) + if err != nil { + t.Fatal(err) + } + trees[i] = tr + } + runtime.GC() + runtime.ReadMemStats(&m1) + runtime.KeepAlive(trees) + return (int64(m1.HeapAlloc) - int64(m0.HeapAlloc)) / int64(n) +} + +// A namespace URI was the xmlns attribute's value as the tokeniser returned +// it, a string in its arena; the tree keeps it in every name it resolves, so +// one declaration kept a whole arena block alive per document. Measured on +// the DocBook corpus: 31.6 MB retained for 591 trees before, 12.3 MB after. +// The two documents differ only in whether one attribute is a declaration. +func TestNamespaceURIDoesNotPinArena(t *testing.T) { + if raceEnabled { + t.Skip("heap sizes are not stable under -race") + } + // Attribute values fill the arena; each is copied into the tree, so + // neither document should keep the arena itself. + var vals string + for i := range 9 { + vals += ` a` + string(rune('0'+i)) + `="` + strings.Repeat("v", 900) + `"` + } + withNS := `` + without := `` + const n = 200 + plain := retainedPerTree(t, without, ParseOptions{}, n) + ns := retainedPerTree(t, withNS, ParseOptions{}, n) + t.Logf("retained per tree: %d bytes without a declaration, %d with", plain, ns) + if ns-plain > 2<<10 { + t.Errorf("a namespace declaration costs %d bytes per kept tree, want under 2 KB", ns-plain) + } +} diff --git a/xdm/wellformed.go b/xdm/wellformed.go index caaa9ffe..cb68acc1 100644 --- a/xdm/wellformed.go +++ b/xdm/wellformed.go @@ -281,7 +281,10 @@ func (s *nsScope) declare(prefix, uri string) { s.prev = append(s.prev, s.top[prefix]-1) s.top[prefix] = len(s.prev) } - s.b = append(s.b, nsBinding{prefix, uri}) + // The URI is copied out of the tokeniser's arena: the tree keeps it in + // every name and frame it resolves, and one arena string reachable from a + // tree keeps its whole 32 KB block alive. + s.b = append(s.b, nsBinding{prefix, strings.Clone(uri)}) s.own++ } diff --git a/xdm/wellformed_scope_test.go b/xdm/wellformed_scope_test.go index 74dddd76..ef82f078 100644 --- a/xdm/wellformed_scope_test.go +++ b/xdm/wellformed_scope_test.go @@ -215,8 +215,8 @@ func TestTagScopeDoesNotAllocate(t *testing.T) { t.Fatal(err) } ns.pop(ns.own) - }); n != 0 { - t.Errorf("validateStartElement under 16 inherited bindings allocated %.0f times, want 0", n) + }); n > 1 { // the one is the declared URI's copy out of the arena (nsScope.declare) + t.Errorf("validateStartElement under 16 inherited bindings allocated %.0f times, want 1", n) } } From a5b385de7d981a090af9167a4d455256de432017 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:12:44 -0400 Subject: [PATCH 03/98] xslt: hold fn:transform cache resolvers weakly A fresh resolver per transform left up to nestedCacheSize dead nested compiles alive (49 MB on DocBook section.001 over 20 resolvers). The key now holds the resolvers as weak pointers and put drops entries whose resolver has been collected (3.3 MB, the live resolver's entries). --- xslt/fntransformcache.go | 33 ++++++++++++++++++++++--- xslt/fntransformcache_test.go | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/xslt/fntransformcache.go b/xslt/fntransformcache.go index f4c100f2..8ef5f975 100644 --- a/xslt/fntransformcache.go +++ b/xslt/fntransformcache.go @@ -8,6 +8,7 @@ import ( "slices" "strings" "sync" + "weak" "github.com/knroy/go-xml/v2/xdm" "github.com/knroy/go-xml/v2/xpath" @@ -46,11 +47,29 @@ type nestedEntry struct { // stylesheet-node, keyed by identity: XDM trees are immutable, and holding // the pointer in the key keeps the tree alive, so the address cannot be // reused by another tree while the entry exists. docs and pkgs are the -// resolvers the nested compilation inherits, likewise held by the key. +// resolvers the nested compilation inherits, held weakly: a weak pointer +// keeps its identity after the resolver is collected, so a new resolver at +// the same address never matches, and a caller that makes a fresh resolver +// per transform does not keep every compile it made alive (put drops them). type nestedKey struct { opts string node *xdm.Node - docs, pkgs any + docs, pkgs weak.Pointer[byte] +} + +// dead reports whether a resolver the key names has been collected, so the +// entry can never be hit again. +func (k nestedKey) dead() bool { + gone := func(p weak.Pointer[byte]) bool { return p != weak.Pointer[byte]{} && p.Value() == nil } + return gone(k.docs) || gone(k.pkgs) +} + +// weakResolver is the weak identity of a resolver keyable accepted. +func weakResolver(v any) weak.Pointer[byte] { + if v == nil { + return weak.Pointer[byte]{} + } + return weak.Make((*byte)(reflect.ValueOf(v).UnsafePointer())) } func (c *nestedCache) get(k nestedKey) *Stylesheet { @@ -77,6 +96,14 @@ func (c *nestedCache) put(k nestedKey, s *Stylesheet) { if c.byKey == nil { c.byKey = map[nestedKey]*list.Element{} } + for e := c.order.Front(); e != nil; { + next := e.Next() + if old := e.Value.(*nestedEntry).key; old.dead() { + c.order.Remove(e) + delete(c.byKey, old) + } + e = next + } c.byKey[k] = c.order.PushFront(&nestedEntry{k, s}) if c.order.Len() > nestedCacheSize { old := c.order.Back() @@ -121,7 +148,7 @@ func nestedCacheKey(ctx *xpath.Context, rt transformCaller, opts *xdm.MapItem) ( if !keyable(mr) || !keyable(rt.pkgs) { return nestedKey{}, false } - k := nestedKey{docs: mr, pkgs: rt.pkgs} + k := nestedKey{docs: weakResolver(mr), pkgs: weakResolver(rt.pkgs)} var b strings.Builder field := func(s string) { fmt.Fprintf(&b, "%d:%s;", len(s), s) } field(ctx.StaticBaseURI()) diff --git a/xslt/fntransformcache_test.go b/xslt/fntransformcache_test.go index 7ce9f7e0..629d3d8f 100644 --- a/xslt/fntransformcache_test.go +++ b/xslt/fntransformcache_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + goruntime "runtime" "strings" "sync" "testing" @@ -156,6 +157,50 @@ func TestTransformCache(t *testing.T) { } }) + // A fresh resolver per transform, the natural pattern in a service: the + // entries keyed on resolvers that have been collected are dropped, so the + // cache holds only the live one rather than nestedCacheSize dead compiles. + t.Run("dead resolver dropped", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "inner.xsl") + if err := os.WriteFile(path, []byte(cacheInner), 0o644); err != nil { + t.Fatal(err) + } + src := ` + + + + + ` + sd, err := xdm.ParseString(src, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + st, err := Compile(sd.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + run := func() { + r, err := NewFileResolver(dir) + if err != nil { + t.Fatal(err) + } + if _, err := st.Transform(context.Background(), nil, TransformOptions{ + InitialTemplate: "go", Documents: r}); err != nil { + t.Fatal(err) + } + } + for range 5 { + run() + } + goruntime.GC() + run() + if c, e := cacheState(st); c != 6 || e != 1 { + t.Errorf("compiles=%d entries=%d over six resolvers, want 6 and 1", c, e) + } + }) + // A compile failure is never stored: each call recompiles and reports // the stylesheet's own static error. t.Run("error not cached", func(t *testing.T) { From 8dcb07fb6a37399d471276c1d454a0078af779ce Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:13:45 -0400 Subject: [PATCH 04/98] dtd: render the error path only for a recorded failure walk concatenated parentPath + "/" + name at every element, which is quadratic in depth. fail now renders the path from the element's ancestors, and an IDREF keeps its element instead of a path. Text is unchanged. Valid 990-deep chain of 4 KB names: 2015 MB -> 0.00 MB allocated. --- dtd/path_alloc_test.go | 52 +++++++++++++++++++++++++ dtd/validate.go | 88 +++++++++++++++++++++++++----------------- 2 files changed, 105 insertions(+), 35 deletions(-) create mode 100644 dtd/path_alloc_test.go diff --git a/dtd/path_alloc_test.go b/dtd/path_alloc_test.go new file mode 100644 index 00000000..c44c1ae1 --- /dev/null +++ b/dtd/path_alloc_test.go @@ -0,0 +1,52 @@ +package dtd + +import ( + "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// The path is rendered only for a recorded failure. Concatenated at every +// element it cost the square of the depth: a valid 990-deep chain of 4 KB +// names allocated 2 GB. The text of a reported path is unchanged, including +// for an IDREF, which is resolved after the walk has left the element. +func TestPathBuiltOnlyForFailures(t *testing.T) { + d, err := Parse(` + ]>`) + if err != nil { + t.Fatal(err) + } + tree, err := xdm.ParseString(``, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + const want = "2 validity errors:\n" + + " /r/a/b: attribute x is #FIXED \"0\" but is \"1\"\n" + + " /r/a/c: IDREF \"nope\" matches no ID in the document" + if err := Validate(tree.Root, d, Options{}); err == nil || err.Error() != want { + t.Fatalf("got\n%v\nwant\n%s", err, want) + } + + nm := "n" + strings.Repeat("x", 4095) + d, err = Parse(`]>`) + if err != nil { + t.Fatal(err) + } + tree, err = xdm.ParseString(strings.Repeat("<"+nm+">", 990)+strings.Repeat("", 990), xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var m0, m1 runtime.MemStats + runtime.ReadMemStats(&m0) + err = Validate(tree.Root, d, Options{}) + runtime.ReadMemStats(&m1) + if err != nil { + t.Fatal(err) + } + if got := m1.TotalAlloc - m0.TotalAlloc; got > 16<<20 { + t.Errorf("validating a valid deep chain allocated %d MB, want under 16 MB", got>>20) + } +} diff --git a/dtd/validate.go b/dtd/validate.go index 6e6692d8..4dce818a 100644 --- a/dtd/validate.go +++ b/dtd/validate.go @@ -2,6 +2,7 @@ package dtd import ( "fmt" + "slices" "strings" "github.com/knroy/go-xml/v2/xdm" @@ -133,7 +134,8 @@ func Validate(doc *xdm.Node, d *DTD, opts Options) error { } } } - v.walk(root, "") + v.root = root + v.walk(root) v.checkIDRefs() if len(v.errs) == 0 { return nil @@ -161,40 +163,56 @@ type validator struct { // refs are IDREF values and where they appeared, checked once the whole // document has been read — a reference may point forward. refs []idref + // root is the element the walk started from, where a path begins. + root *xdm.Node } type idref struct { value string - path string + el *xdm.Node } -func (v *validator) fail(path, format string, args ...any) { +func (v *validator) fail(el *xdm.Node, format string, args ...any) { if v.max > 0 && len(v.errs) >= v.max { return } - v.errs = append(v.errs, &Error{Path: path, Message: fmt.Sprintf(format, args...)}) + v.errs = append(v.errs, &Error{Path: v.pathOf(el), Message: fmt.Sprintf(format, args...)}) } -func (v *validator) walk(el *xdm.Node, parentPath string) { - path := parentPath + "/" + el.Name().Local - v.checkContent(el, path) - v.checkAttributes(el, path) +// pathOf renders the path from the root to el, as "/root/child". It is built +// only for a recorded failure: concatenating it at every element cost the +// square of the depth, 2 GB allocated for a valid 990-deep chain of 4 KB names. +func (v *validator) pathOf(el *xdm.Node) string { + var names []string + for n := el; ; n = n.Parent() { + names = append(names, n.Name().Local) + if n == v.root { + break + } + } + slices.Reverse(names) + return "/" + strings.Join(names, "/") +} + +func (v *validator) walk(el *xdm.Node) { + v.checkContent(el) + v.checkAttributes(el) for c := range el.Children() { if c.Kind() == xdm.KindElement { - v.walk(c, path) + v.walk(c) } } } // checkContent applies the element's content model. -func (v *validator) checkContent(el *xdm.Node, path string) { +func (v *validator) checkContent(el *xdm.Node) { decl, ok := v.dtd.Elements[el.Name().Local] if !ok { // An undeclared element is a validity error in a DTD-validated // document — unlike XSD, where a wildcard may admit it. A caller // working with a deliberately partial internal subset can say so. if !v.allowUndec { - v.fail(path, "element %s is not declared", el.Name().Local) + v.fail(el, "element %s is not declared", el.Name().Local) } return } @@ -204,7 +222,7 @@ func (v *validator) checkContent(el *xdm.Node, path string) { case ContentEmpty: for c := range el.Children() { if c.Kind() == xdm.KindElement || (c.Kind() == xdm.KindText && strings.TrimSpace(c.Value()) != "") { - v.fail(path, "element %s is declared EMPTY but has content", + v.fail(el, "element %s is declared EMPTY but has content", el.Name().Local) return } @@ -212,7 +230,7 @@ func (v *validator) checkContent(el *xdm.Node, path string) { case ContentMixed: for c := range el.Children() { if c.Kind() == xdm.KindElement && !decl.Mixed[c.Name().Local] { - v.fail(path, "element %s is not permitted in the mixed "+ + v.fail(el, "element %s is not permitted in the mixed "+ "content of %s", c.Name().Local, el.Name().Local) } } @@ -221,14 +239,14 @@ func (v *validator) checkContent(el *xdm.Node, path string) { // unexpected elements: "(a, b)" does not admit text between them. for c := range el.Children() { if c.Kind() == xdm.KindText && strings.TrimSpace(c.Value()) != "" { - v.fail(path, "element %s has element-only content but "+ + v.fail(el, "element %s has element-only content but "+ "contains character data", el.Name().Local) break } } m, err := v.matcherFor(decl) if err != nil { - v.fail(path, "content model of %s: %v", el.Name().Local, err) + v.fail(el, "content model of %s: %v", el.Name().Local, err) return } var names []xdm.QName @@ -239,10 +257,10 @@ func (v *validator) checkContent(el *xdm.Node, path string) { } if ok, at := m.Match(names); !ok { if at < len(names) { - v.fail(path, "element %s is not permitted here in the "+ + v.fail(el, "element %s is not permitted here in the "+ "content of %s", names[at].Local, el.Name().Local) } else { - v.fail(path, "the content of %s is incomplete", el.Name().Local) + v.fail(el, "the content of %s is incomplete", el.Name().Local) } } } @@ -261,7 +279,7 @@ func (v *validator) matcherFor(decl *Element) (*xsd.SequenceMatcher, error) { } // checkAttributes applies the ATTLIST declarations for an element. -func (v *validator) checkAttributes(el *xdm.Node, path string) { +func (v *validator) checkAttributes(el *xdm.Node) { decls := v.dtd.Attributes[el.Name().Local] present := map[string]string{} for a := range el.Attrs() { @@ -280,16 +298,16 @@ func (v *validator) checkAttributes(el *xdm.Node, path string) { switch d.Default { case AttrRequired: if !have { - v.fail(path, "required attribute %s is missing", d.Name) + v.fail(el, "required attribute %s is missing", d.Name) continue } case AttrFixed: if have && val != d.Value { - v.fail(path, "attribute %s is #FIXED %q but is %q", + v.fail(el, "attribute %s is #FIXED %q but is %q", d.Name, d.Value, val) } case AttrInvalid: - v.fail(path, "attribute %s has an unrecognised default "+ + v.fail(el, "attribute %s has an unrecognised default "+ "declaration %s, so its presence was not checked", d.Name, d.Value) } @@ -297,7 +315,7 @@ func (v *validator) checkAttributes(el *xdm.Node, path string) { continue } if len(d.Enum) > 0 && !containsString(d.Enum, val) { - v.fail(path, "attribute %s = %q is not one of %s", + v.fail(el, "attribute %s = %q is not one of %s", d.Name, val, strings.Join(d.Enum, ", ")) } switch d.Type { @@ -309,34 +327,34 @@ func (v *validator) checkAttributes(el *xdm.Node, path string) { // new here, and it is a property of the DECLARATION rather than // of this element's value — so it is reported once per // declaration, not once per occurrence. - v.checkNotationDecl(path, d) + v.checkNotationDecl(el, d) case "ENTITY": // §3.3.1: the value must be the name of an unparsed entity — // declared, and declared with an NDATA notation. A parsed // entity's name is as wrong here as an undeclared one. - v.checkEntityValue(path, d.Name, val) + v.checkEntityValue(el, d.Name, val) case "ENTITIES": // The same rule over a whitespace-separated list. An empty list // is not a Names, so it fails the type as surely as an unknown // name would. names := strings.Fields(val) if len(names) == 0 { - v.fail(path, "attribute %s is ENTITIES but is empty, which "+ + v.fail(el, "attribute %s is ENTITIES but is empty, which "+ "names no unparsed entity", d.Name) } for _, n := range names { - v.checkEntityValue(path, d.Name, n) + v.checkEntityValue(el, d.Name, n) } case "ID": if v.ids[val] { - v.fail(path, "duplicate ID %q", val) + v.fail(el, "duplicate ID %q", val) } v.ids[val] = true case "IDREF": - v.refs = append(v.refs, idref{val, path}) + v.refs = append(v.refs, idref{val, el}) case "IDREFS": for _, r := range strings.Fields(val) { - v.refs = append(v.refs, idref{r, path}) + v.refs = append(v.refs, idref{r, el}) } case "CDATA", "NMTOKEN", "NMTOKENS", "ENUMERATION": // Declared, and either unconstrained (CDATA) or constrained by @@ -348,7 +366,7 @@ func (v *validator) checkAttributes(el *xdm.Node, path string) { // Without this branch an unrecognised type — "IDREFF" for // "IDREFS" — falls through to no check at all, and the caller // cannot tell an attribute that passed from one never examined. - v.fail(path, "attribute %s is declared with the unrecognised "+ + v.fail(el, "attribute %s is declared with the unrecognised "+ "type %s, so its value was not checked", d.Name, d.Type) } } @@ -361,7 +379,7 @@ func (v *validator) checkAttributes(el *xdm.Node, path string) { // It is reported once per declaration rather than once per element carrying // the attribute, because the fault is in the DTD and repeating it for every // instance would bury the finding under its own copies. -func (v *validator) checkNotationDecl(path string, d *Attribute) { +func (v *validator) checkNotationDecl(el *xdm.Node, d *Attribute) { if v.partial { // The declarations were never all read, so a name absent from // Notations may simply be in the half that was not fetched. @@ -376,20 +394,20 @@ func (v *validator) checkNotationDecl(path string, d *Attribute) { v.notationsChecked[key] = true for _, n := range d.Enum { if !v.dtd.Notations[n] { - v.fail(path, "attribute %s permits the notation %s, which no "+ + v.fail(el, "attribute %s permits the notation %s, which no "+ " declares", d.Name, n) } } } // checkEntityValue applies §3.3.1's Entity Name constraint to one name. -func (v *validator) checkEntityValue(path, attr, name string) { +func (v *validator) checkEntityValue(el *xdm.Node, attr, name string) { if v.partial { // As for notations: half a DTD cannot say an entity is undeclared. return } if !v.dtd.Unparsed[name] { - v.fail(path, "attribute %s = %q names no unparsed entity", attr, name) + v.fail(el, "attribute %s = %q names no unparsed entity", attr, name) } } @@ -398,7 +416,7 @@ func (v *validator) checkEntityValue(path, attr, name string) { func (v *validator) checkIDRefs() { for _, r := range v.refs { if !v.ids[r.value] { - v.fail(r.path, "IDREF %q matches no ID in the document", r.value) + v.fail(r.el, "IDREF %q matches no ID in the document", r.value) } } } From ba9e7a044005b2901141d625983c1b939363652f Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:13:58 -0400 Subject: [PATCH 05/98] xquery: share a FLWOR stream's tuple names slice A clause binding one name onto every tuple copied the names slice per tuple, and "for" wrapped each item in a fresh one-item sequence. Tuples of one stream now share the extended names slice, replacing a binding shares the names, and "for" and the join bind a capped subslice of the binding sequence. XMark q1-q20 eval: 194.9 -> 156.7 MB. --- xquery/flwor.go | 43 ++++++++++++++++++++++++++++++-------- xquery/flwor_alloc_test.go | 40 +++++++++++++++++++++++++++++++++++ xquery/join.go | 5 ++++- 3 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 xquery/flwor_alloc_test.go diff --git a/xquery/flwor.go b/xquery/flwor.go index 001bfb57..9f0e0b56 100644 --- a/xquery/flwor.go +++ b/xquery/flwor.go @@ -2,6 +2,7 @@ package xquery import ( "fmt" + "slices" "sort" "strings" @@ -50,11 +51,12 @@ type tuple struct { // Copying rather than mutating is what makes "for" correct: the body of a // downstream clause may capture the tuple (an inline function closing over // $x, a grouped partition holding its members), and a tuple mutated in place -// would make every capture observe the last value. +// would make every capture observe the last value. The names are never +// written once a tuple holds them, so replacing a binding shares them. func (t tuple) bind(name xdm.QName, val xdm.Sequence) tuple { for i, n := range t.names { if n == name { - out := t.clone() + out := tuple{names: t.names, vals: slices.Clone(t.vals)} out.vals[i] = val return out } @@ -66,11 +68,31 @@ func (t tuple) bind(name xdm.QName, val xdm.Sequence) tuple { return out } -func (t tuple) clone() tuple { - return tuple{ - names: append([]xdm.QName(nil), t.names...), - vals: append([]xdm.Sequence(nil), t.vals...), +// nameExt is the names slice a clause last extended and what it extended it +// to. A clause binds one name onto every tuple of its stream, and the tuples +// of a stream mostly share one names slice, so the extended slice is made +// once per shared input slice rather than once per tuple. +type nameExt struct{ from, to []xdm.QName } + +func (x *nameExt) extend(names []xdm.QName, name xdm.QName) []xdm.QName { + if x.to == nil || len(names) != len(x.from) || + (len(names) > 0 && &names[0] != &x.from[0]) { + x.from = names + x.to = append(append(make([]xdm.QName, 0, len(names)+1), names...), name) + } + return x.to +} + +// bindx is bind for a clause binding name onto every tuple of a stream, with +// the extended names slice shared through x. +func (t tuple) bindx(x *nameExt, name xdm.QName, val xdm.Sequence) tuple { + if slices.Contains(t.names, name) { + return t.bind(name, val) } + vals := make([]xdm.Sequence, len(t.vals)+1) + copy(vals, t.vals) + vals[len(t.vals)] = val + return tuple{names: x.extend(t.names, name), vals: vals} } func (t tuple) lookup(name xdm.QName) (xdm.Sequence, bool) { @@ -220,6 +242,7 @@ type forClause struct { func (c *forClause) apply(in []tuple, ctx *evalContext) ([]tuple, error) { var out []tuple + var x nameExt for _, t := range in { seq, err := c.seq.eval(t.sub(ctx)) if err != nil { @@ -246,8 +269,9 @@ func (c *forClause) apply(in []tuple, ctx *evalContext) ([]tuple, error) { out = append(out, n) continue } - for i, it := range seq { - n := t.bind(c.name, xdm.One(it)) + out = slices.Grow(out, len(seq)) + for i := range seq { + n := t.bindx(&x, c.name, seq[i:i+1:i+1]) if c.hasPos { n = n.bind(c.pos, xdm.One(xdm.NewInteger(int64(i+1)))) } @@ -268,12 +292,13 @@ type letClause struct { func (c *letClause) apply(in []tuple, ctx *evalContext) ([]tuple, error) { out := make([]tuple, 0, len(in)) + var x nameExt for _, t := range in { seq, err := c.seq.eval(t.sub(ctx)) if err != nil { return nil, err } - out = append(out, t.bind(c.name, seq)) + out = append(out, t.bindx(&x, c.name, seq)) } return out, nil } diff --git a/xquery/flwor_alloc_test.go b/xquery/flwor_alloc_test.go new file mode 100644 index 00000000..7ad4d145 --- /dev/null +++ b/xquery/flwor_alloc_test.go @@ -0,0 +1,40 @@ +package xquery + +import ( + "testing" + + "github.com/knroy/go-xml/v2/xpath" +) + +// flworAllocs is the allocations one evaluation of src makes, after checking +// that it returns want items. +func flworAllocs(t *testing.T, src string, want int) float64 { + t.Helper() + q, err := Compile(src, Options{}) + if err != nil { + t.Fatal(err) + } + ctx := xpath.NewContext(nil, xpath.Builtins()) + var n int + allocs := testing.AllocsPerRun(3, func() { + got, err := q.Eval(ctx) + if err != nil { + t.Fatal(err) + } + n = len(got) + }) + if n != want { + t.Fatalf("%d items, want %d", n, want) + } + t.Logf("%.0f allocations", allocs) + return allocs +} + +// TestFLWORTupleNamesShared pins that the tuples a clause produces share one +// names slice and that "for" binds each item without wrapping it. Measured: +// 35,300 allocations before, 25,290 after. +func TestFLWORTupleNamesShared(t *testing.T) { + if a := flworAllocs(t, `for $a in 1 to 2000 let $b := $a return $b`, 2000); a > 30000 { + t.Errorf("%.0f allocations, want at most 30,000", a) + } +} diff --git a/xquery/join.go b/xquery/join.go index 84f4cdad..68afd31c 100644 --- a/xquery/join.go +++ b/xquery/join.go @@ -2,6 +2,7 @@ package xquery import ( "math" + "slices" "sort" "github.com/knroy/go-xml/v2/xdm" @@ -196,6 +197,7 @@ func (c *joinClause) join(in []tuple, ctx *evalContext) (out []tuple, n int, ok if state == nil || state.released { state = &joinState{} // this apply only } + var x nameExt for _, t := range in { if ctx.xp.Err() != nil { return nil, 0, false @@ -222,8 +224,9 @@ func (c *joinClause) join(in []tuple, ctx *evalContext) (out []tuple, n int, ok return nil, 0, false } n += len(hits) + out = slices.Grow(out, len(hits)) for _, i := range hits { - out = append(out, t.bind(c.f.name, e.items[i:i+1:i+1])) + out = append(out, t.bindx(&x, c.f.name, e.items[i:i+1:i+1])) } } return out, n, true From bd865e4c713fc384a6259375c42679b49d6c3337 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:14:19 -0400 Subject: [PATCH 06/98] dtd: clip values quoted in validity messages fail kept a #FIXED, enumerated or ID value whole in every error. It now clips string and error arguments and the message to 1 KB. 200 mismatches against a 1 MB #FIXED value: 104.6 MB live / 317 MB allocated -> ~0 / 0.49 MB. --- dtd/message_clip_test.go | 36 ++++++++++++++++++++++++++++++++++++ dtd/validate.go | 23 ++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 dtd/message_clip_test.go diff --git a/dtd/message_clip_test.go b/dtd/message_clip_test.go new file mode 100644 index 00000000..f9de1fbe --- /dev/null +++ b/dtd/message_clip_test.go @@ -0,0 +1,36 @@ +package dtd + +import ( + "errors" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// A value quoted in a message is clipped: 200 mismatches against a 1 MB +// #FIXED value kept 105 MB in the errors. The message keeps its prefix. +func TestValidityMessagesClipLongValues(t *testing.T) { + big := strings.Repeat("v", 1<<20) + d, err := Parse(` + ]>`) + if err != nil { + t.Fatal(err) + } + tree, err := xdm.ParseString(""+strings.Repeat(``, 200)+"", xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var errs *Errors + if err := Validate(tree.Root, d, Options{}); !errors.As(err, &errs) || len(errs.Errors) == 0 { + t.Fatalf("expected Errors, got %v", err) + } + for _, e := range errs.Errors { + if len(e.Message) > 1024+len("...") { + t.Fatalf("message is %d bytes, want at most 1027", len(e.Message)) + } + if !strings.HasPrefix(e.Message, `attribute a is #FIXED "vvvv`) || !strings.HasSuffix(e.Message, "...") { + t.Fatalf("message lost its prefix or its clip marker: %.200q", e.Message) + } + } +} diff --git a/dtd/validate.go b/dtd/validate.go index 4dce818a..8b946bcb 100644 --- a/dtd/validate.go +++ b/dtd/validate.go @@ -176,7 +176,28 @@ func (v *validator) fail(el *xdm.Node, format string, args ...any) { if v.max > 0 && len(v.errs) >= v.max { return } - v.errs = append(v.errs, &Error{Path: v.pathOf(el), Message: fmt.Sprintf(format, args...)}) + // A #FIXED or enumerated value comes from the DTD or the document and + // may be megabytes; 200 mismatches against a 1 MB #FIXED value kept + // 105 MB. Arguments are clipped before formatting, the message after. + for i, a := range args { + switch x := a.(type) { + case string: + args[i] = clip(x) + case error: + args[i] = clip(x.Error()) + } + } + v.errs = append(v.errs, &Error{Path: v.pathOf(el), Message: clip(fmt.Sprintf(format, args...))}) +} + +// clip shortens s to 1 KB, marking the cut with "..." as the xsd package +// does. The concatenation copies, so the result never pins the original. +func clip(s string) string { + const max = 1024 + if len(s) <= max { + return s + } + return s[:max] + "..." } // pathOf renders the path from the root to el, as "/root/child". It is built From f890740275485db74c9d5e9e93046519af1de1ab Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:14:39 -0400 Subject: [PATCH 07/98] xquery: return a FLWOR's value without rebuilding it "let $l := for ..." sent the FLWOR's result through a top-level builder, which appended every item a second time. flworNode is now a valueNode and answers with the sequence itself. XMark q1-q20 eval: 156.7 -> 142.7 MB. --- xquery/flwor.go | 6 ++++++ xquery/flwor_alloc_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/xquery/flwor.go b/xquery/flwor.go index 9f0e0b56..ccfef340 100644 --- a/xquery/flwor.go +++ b/xquery/flwor.go @@ -709,6 +709,12 @@ func (n *flworNode) eval(out *builderRef, ctx *evalContext) error { return n.f.evalNode(out, ctx) } +// sequence makes a flworNode a valueNode. Built at the top level, its value +// would come back out of the builder unchanged, but appended a second time. +func (n *flworNode) sequence(ctx *evalContext) (xdm.Sequence, error) { + return n.f.eval(ctx) +} + type quantifiedNode struct{ q *quantified } func (n *quantifiedNode) eval(out *builderRef, ctx *evalContext) error { diff --git a/xquery/flwor_alloc_test.go b/xquery/flwor_alloc_test.go index 7ad4d145..f819bc98 100644 --- a/xquery/flwor_alloc_test.go +++ b/xquery/flwor_alloc_test.go @@ -1,6 +1,7 @@ package xquery import ( + "runtime" "testing" "github.com/knroy/go-xml/v2/xpath" @@ -38,3 +39,40 @@ func TestFLWORTupleNamesShared(t *testing.T) { t.Errorf("%.0f allocations, want at most 30,000", a) } } + +// flworBytes is the bytes one evaluation of src allocates, the least of three +// runs, after checking that it returns want items. +func flworBytes(t *testing.T, src string, want int) uint64 { + t.Helper() + q, err := Compile(src, Options{}) + if err != nil { + t.Fatal(err) + } + ctx := xpath.NewContext(nil, xpath.Builtins()) + least := ^uint64(0) + for range 3 { + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + got, err := q.Eval(ctx) + runtime.ReadMemStats(&after) + if err != nil { + t.Fatal(err) + } + if len(got) != want { + t.Fatalf("%d items, want %d", len(got), want) + } + least = min(least, after.TotalAlloc-before.TotalAlloc) + } + t.Logf("%d bytes", least) + return least +} + +// TestFLWORLetValueNotRebuilt pins that a FLWOR bound by "let" is its own +// value rather than appended item by item through a builder. +// Measured: 29.4 MB before, 25.1 MB after. +func TestFLWORLetValueNotRebuilt(t *testing.T) { + b := flworBytes(t, `let $l := for $a in 1 to 50000 return $a return $l`, 50000) + if b > 27_000_000 { + t.Errorf("%d bytes, want at most 27,000,000", b) + } +} From 9e13f70e201d50886a4534519d1988aa1d8afdc1 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:14:51 -0400 Subject: [PATCH 08/98] xmltok: size the string arena's first block to the input Every document allocated a 32 KB arena block. It was kept at that size because a smaller retained heap ran the collector more often, but the retention was the namespace-URI pin, now gone. DocBook corpus: 37.6 -> 19.6 MB a parse pass, GC cycles 13.4 -> 6.8. --- internal/xmltok/xmltok.go | 13 +++++++++---- xdm/parsewindow_test.go | 10 +++++----- xdm/wellformed.go | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/internal/xmltok/xmltok.go b/internal/xmltok/xmltok.go index cf987ee6..1082e91c 100644 --- a/internal/xmltok/xmltok.go +++ b/internal/xmltok/xmltok.go @@ -162,7 +162,8 @@ type Decoder struct { // Size, when set, is about how many bytes the stream holds. A long text // run that outgrows the scratch sizes it from what is left, rather than - // growing it by append's steps window by window. + // growing it by append's steps window by window, and the string arena's + // first block is no larger than it. Size int src io.Reader @@ -215,9 +216,7 @@ func NewDecoder(r io.Reader) *Decoder { // NewDecoderSize is NewDecoder for a stream known to hold about size bytes: // its read window is no larger than the input needs, where a small document -// otherwise paid for 4 KB. The string arena is not sized down: a smaller -// arena retains less heap per tree, which made the collector run more often -// (docs/profiling.md, "Measured and rejected"). +// otherwise paid for 4 KB. The string arena's first block is sized by Size. func NewDecoderSize(r io.Reader, size int) *Decoder { return &Decoder{Strict: true, src: r, buf: make([]byte, 0, min(max(size, minWindow), bufSize))} } @@ -463,6 +462,12 @@ func (d *Decoder) startTag() (Token, error) { if !ok { return nil, d.err } + if d.arena.buf == nil && d.Size > 0 { + // A document smaller than a block gets a first block its own + // size; the values of a larger one, or what entities expand + // to, go on to full blocks. + d.arena.buf = make([]byte, 0, min(d.Size, arenaBlock)) + } attrs = append(attrs, Attr{an, d.arena.String(v)}) } d.attrs = attrs diff --git a/xdm/parsewindow_test.go b/xdm/parsewindow_test.go index a19f99c1..a2af0724 100644 --- a/xdm/parsewindow_test.go +++ b/xdm/parsewindow_test.go @@ -39,9 +39,9 @@ func TestSizedParseMatchesUnsized(t *testing.T) { } // A small document's parse allocated three 4 KB read windows whatever its -// size. Measured on this document: 47,576 bytes a parse with them, 36,942 -// with windows sized to it (most of the rest is the 32 KB string arena, -// which is kept: see docs/profiling.md, "Measured and rejected"). +// size, and a 32 KB string arena block. Measured on this document: 47,576 +// bytes a parse with them, 36,942 with windows sized to it, 4,424 with the +// arena's first block sized to it too. func TestSmallParseAllocatesLittle(t *testing.T) { if raceEnabled { t.Skip("allocation sizes are not stable under -race") @@ -58,7 +58,7 @@ func TestSmallParseAllocatesLittle(t *testing.T) { runtime.ReadMemStats(&m1) per := (m1.TotalAlloc - m0.TotalAlloc) / n t.Logf("%d bytes per parse", per) - if per > 42<<10 { - t.Errorf("%d bytes per parse of a %d-byte document, want at most 42 KB", per, len(doc)) + if per > 12<<10 { + t.Errorf("%d bytes per parse of a %d-byte document, want at most 12 KB", per, len(doc)) } } diff --git a/xdm/wellformed.go b/xdm/wellformed.go index cb68acc1..700d8362 100644 --- a/xdm/wellformed.go +++ b/xdm/wellformed.go @@ -283,7 +283,7 @@ func (s *nsScope) declare(prefix, uri string) { } // The URI is copied out of the tokeniser's arena: the tree keeps it in // every name and frame it resolves, and one arena string reachable from a - // tree keeps its whole 32 KB block alive. + // tree keeps its whole block alive. s.b = append(s.b, nsBinding{prefix, strings.Clone(uri)}) s.own++ } From a8d6e7b5c4b905ec441414b591e0c97984746c8c Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:15:10 -0400 Subject: [PATCH 09/98] xquery: install the query's XPath version once per evaluation A query compiled as 3.1 under a 2.0 context made every expression evaluation copy the context to install the version. prepare installs it once. XMark q1-q20 eval: 142.7 -> 133.1 MB. --- xquery/flwor_alloc_test.go | 10 ++++++++++ xquery/xquery.go | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/xquery/flwor_alloc_test.go b/xquery/flwor_alloc_test.go index f819bc98..eaf27ecb 100644 --- a/xquery/flwor_alloc_test.go +++ b/xquery/flwor_alloc_test.go @@ -76,3 +76,13 @@ func TestFLWORLetValueNotRebuilt(t *testing.T) { t.Errorf("%d bytes, want at most 27,000,000", b) } } + +// TestEvalInstallsQueryVersion pins that evaluating a query under a context +// of another XPath version installs the query's once, rather than each +// expression copying the context to install it. Measured: 36,526 +// allocations before, 34,527 after. +func TestEvalInstallsQueryVersion(t *testing.T) { + if a := flworAllocs(t, `for $a in 1 to 2000 return $a + 1`, 2000); a > 35500 { + t.Errorf("%.0f allocations, want at most 35,500", a) + } +} diff --git a/xquery/xquery.go b/xquery/xquery.go index d908ed41..36ef0db0 100644 --- a/xquery/xquery.go +++ b/xquery/xquery.go @@ -420,7 +420,11 @@ func (q *Query) prepare(ctx *xpath.Context) (*xpath.Context, error) { // expression by compileExpr, because xpath models both as static // properties of an expression. Setting the context's base URI as well is // what fn:static-base-uri and fn:doc's relative resolution read. - out := &sub + // + // The version is installed once here for the same reason: every + // expression in the query was compiled under it, and a context that + // disagreed made each of their evaluations copy the context to say so. + out := (&sub).WithVersion(q.sc.xqVersion.xpathVersion()) if q.sc.baseURI != "" { out = out.WithStaticBaseURI(q.sc.baseURI) } From 5d8fcebaa2cfd66b1a3d50123885aa677e41b028 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:15:35 -0400 Subject: [PATCH 10/98] xsd: scan base64Binary bytes in place isBase64Binary copied the value into a []rune, four bytes per input byte. It now makes one byte pass with a four-byte ring for the final quantum. 380 KB value: 1,540,475 B/op, 1 alloc -> 0 allocs. --- xsd/base64_scan_test.go | 38 ++++++++++++++++++++++++++++++++ xsd/validate_attr.go | 48 ++++++++++++++++++++++------------------- 2 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 xsd/base64_scan_test.go diff --git a/xsd/base64_scan_test.go b/xsd/base64_scan_test.go new file mode 100644 index 00000000..3f0ba9ca --- /dev/null +++ b/xsd/base64_scan_test.go @@ -0,0 +1,38 @@ +package xsd + +import ( + "strings" + "testing" +) + +// isBase64Binary scans bytes in place. It converted the value to []rune +// first, four bytes per input byte: 98.6% of validation allocation on a UBL +// invoice with an embedded attachment. Every legal character is ASCII, so the +// byte scan must reject exactly what the rune scan did, non-ASCII included. +func TestBase64BinaryByteScan(t *testing.T) { + for v, want := range map[string]bool{ + "": true, + "QUJD": true, + "QU JD": true, + "QUJD\nQUI=": true, + "QQ==": true, + "QUI=": true, + "QUJ": false, + "QR==": false, + "QUJ=": false, + "Q===": false, + "QU=D": false, + "QQ==QUJD": false, + "QUJé": false, + "éQU": false, + "QUJéQUJD": false, + } { + if got := isBase64Binary(v); got != want { + t.Errorf("isBase64Binary(%q) = %v, want %v", v, got, want) + } + } + long := strings.Repeat(strings.Repeat("QUJD", 19)+"\n", 5000) + "QUI=" + if n := testing.AllocsPerRun(5, func() { isBase64Binary(long) }); n != 0 { + t.Errorf("isBase64Binary allocated %v times on a 380 KB value, want 0", n) + } +} diff --git a/xsd/validate_attr.go b/xsd/validate_attr.go index 7d225f06..4eb590d2 100644 --- a/xsd/validate_attr.go +++ b/xsd/validate_attr.go @@ -623,36 +623,40 @@ func isBase64Binary(v string) bool { b16 = "AEIMQUYcgkosw048" b04 = "AQgw" ) - isB64 := func(r rune) bool { - return r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' || - r >= '0' && r <= '9' || r == '+' || r == '/' + isB64 := func(c byte) bool { + return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || + c >= '0' && c <= '9' || c == '+' || c == '/' } - // Strip whitespace first. The grammar is stated over the significant - // characters, and a quantum may be split across a line break. - buf := make([]rune, 0, len(v)) - for _, r := range v { - switch r { + // One pass over the bytes, skipping whitespace: the grammar is stated + // over the significant characters, and a quantum may be split across a + // line break. Every legal character is ASCII, so a byte pass rejects + // exactly what a rune pass would. The last four significant characters + // are held in a ring until the next arrives, because only the final + // quantum admits padding. + var last [4]byte + n := 0 + for i := 0; i < len(v); i++ { + c := v[i] + switch c { case ' ', '\t', '\n', '\r': - default: - buf = append(buf, r) + continue + } + // Every quantum but the last admits no padding at all. + if n >= 4 && !isB64(last[n%4]) { + return false } + last[n%4] = c + n++ } - if len(buf)%4 != 0 { + if n%4 != 0 { return false } - if len(buf) == 0 { + if n == 0 { return true } - // Every quantum but the last admits no padding at all. - for _, r := range buf[:len(buf)-4] { - if !isB64(r) { - return false - } - } - - last := buf[len(buf)-4:] + // n is a multiple of four, so the ring holds the last quantum in order. if !isB64(last[0]) { return false } @@ -661,10 +665,10 @@ func isBase64Binary(v string) bool { return isB64(last[1]) && isB64(last[2]) && isB64(last[3]) case last[2] == '=': // "B04 B64 = =" -- two octets dropped, four bits must be zero. - return strings.ContainsRune(b04, last[1]) + return strings.IndexByte(b04, last[1]) >= 0 default: // "B16 B64 B16 =" -- one octet dropped, two bits must be zero. - return isB64(last[1]) && strings.ContainsRune(b16, last[2]) + return isB64(last[1]) && strings.IndexByte(b16, last[2]) >= 0 } } From 24b963c63d1837a7066cdc7e7caefd9502731162 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:15:41 -0400 Subject: [PATCH 11/98] xquery: reserve a FLWOR's result from its tuple stream The concatenated result of the return expression grew by doubling. It is now reserved for one item per remaining tuple on the first non-empty return. XMark q1-q20 eval: 133.1 -> 125.6 MB. --- xquery/flwor.go | 8 +++++++- xquery/flwor_alloc_test.go | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/xquery/flwor.go b/xquery/flwor.go index ccfef340..40e5e617 100644 --- a/xquery/flwor.go +++ b/xquery/flwor.go @@ -161,7 +161,7 @@ func (f *flwor) eval(ctx *evalContext) (xdm.Sequence, error) { } } var out xdm.Sequence - for _, t := range stream { + for k, t := range stream { seq, err := f.evalReturn(t, ctx) if err != nil { return nil, err @@ -173,6 +173,12 @@ func (f *flwor) eval(ctx *evalContext) (xdm.Sequence, error) { if err := ctx.xp.ChargeItems(len(seq)); err != nil { return nil, err } + // Sized for one item from each tuple still to come, which is the + // usual return, rather than grown by doubling. The tuples are already + // charged, so the reservation is too. + if out == nil && len(seq) > 0 { + out = make(xdm.Sequence, 0, len(seq)+len(stream)-k-1) + } out = append(out, seq...) } return out, nil diff --git a/xquery/flwor_alloc_test.go b/xquery/flwor_alloc_test.go index eaf27ecb..20011127 100644 --- a/xquery/flwor_alloc_test.go +++ b/xquery/flwor_alloc_test.go @@ -86,3 +86,12 @@ func TestEvalInstallsQueryVersion(t *testing.T) { t.Errorf("%.0f allocations, want at most 35,500", a) } } + +// TestFLWORResultReserved pins that a FLWOR's result is sized from its tuple +// stream rather than grown by doubling. Measured: 24.3 MB before, 20.7 MB +// after. +func TestFLWORResultReserved(t *testing.T) { + if b := flworBytes(t, `for $a in 1 to 50000 return $a`, 50000); b > 22_500_000 { + t.Errorf("%d bytes, want at most 22,500,000", b) + } +} From cf65d62a29a04faaa34df0737f955013cd11b36c Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:16:10 -0400 Subject: [PATCH 12/98] xslt: stream the text method, CDATA sections, character-mapped text and comments The text method concatenated the whole string value before writing it (41.7 MB for 8.2 MB of output on xmark); it now writes each text node. writeCData built every section in a builder, mapSegments allocated a segment slice per text node, and comments and PIs were concatenated: all now write straight to the serializer's writer. Output is unchanged. --- xslt/serialize.go | 58 +++++++++++++----- xslt/serialize_streamalloc_test.go | 94 ++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 xslt/serialize_streamalloc_test.go diff --git a/xslt/serialize.go b/xslt/serialize.go index 09a85440..a82f6667 100644 --- a/xslt/serialize.go +++ b/xslt/serialize.go @@ -314,6 +314,19 @@ func serializeTo(w stringWriter, seq xdm.Sequence, opts OutputSettings, charMap } return s.err } + // Without normalisation the string value is written one + // text descendant at a time rather than concatenated first, + // which held a copy of the whole output. Normalisation keeps + // the concatenation: a combining mark at the start of one + // text node composes with the character ending the last. + if s.normalize == nil && (v.Kind() == xdm.KindDocument || v.Kind() == xdm.KindElement) { + for d := range v.Descendants() { + if d.Kind() == xdm.KindText { + s.writeString(s.mapChars(d.Value())) + } + } + continue + } s.writeString(s.normalized(s.mapChars(v.StringValue()))) case *xdm.Atomic: s.writeString(s.normalized(s.mapChars(v.String()))) @@ -496,6 +509,8 @@ type serializer struct { // its single segment. Both callers finish with the slice before asking // again. oneSeg [1]mapSegment + // segBuf backs mapSegments' answer when there is one, on the same terms. + segBuf []mapSegment // encFrom and encLower cache opts.Encoding lower-cased: representable // asks for it on every non-ASCII character, and lower-casing "UTF-8" // allocated each time. encFrom is the spelling the cache was made from. @@ -711,7 +726,9 @@ func (s *serializer) node(n *xdm.Node, depth int) { return } s.indent(depth) - s.writeString("") + s.writeString("") case xdm.KindPI: if err := s.validatePI(n.Name().Local, n.Value()); err != nil { @@ -730,9 +747,11 @@ func (s *serializer) node(n *xdm.Node, depth int) { return } s.indent(depth) - s.writeString("" alone, // which is the same rule the SERE0015 check above is written for: if @@ -1668,30 +1687,36 @@ func (s *serializer) writeCData(text string) { // character the encoding cannot hold — NFD splits "\u00e7" into "c" and a // combining cedilla, and US-ASCII output must then break the section // around the cedilla rather than write it raw. + // + // The section is written to w in runs, as escapeText writes text: the + // function has no error path, so nothing is gained by building it first. text = s.normalized(text) - var sb strings.Builder - sb.WriteString("") { // The section ends at the first "]]>", which the text may itself // contain — so it is split there and the ">" begins a new one. - sb.WriteString("]]]]>") + s.writeString(text[run:i]) + s.writeString("]]]]>") i += 3 + run = i continue } r, size := utf8.DecodeRuneInString(text[i:]) if !s.representable(r) { - sb.WriteString("]]>") - fmt.Fprintf(&sb, "&#%d;", r) - sb.WriteString("") + s.writeCharRef(r) + s.writeString("") - s.writeString(sb.String()) + s.writeString(text[run:]) + s.writeString("]]>") } // representable reports whether the declared encoding can hold this @@ -1785,7 +1810,10 @@ func (s *serializer) mapSegments(text string) []mapSegment { s.oneSeg[0] = mapSegment{text: s.normalized(text)} return s.oneSeg[:] } - var segs []mapSegment + // The slice is the serializer's own, reused for the same reason as + // oneSeg: a fresh one per text node and attribute value was the whole + // allocation cost of serialising with a character map. + segs := s.segBuf[:0] run := 0 for i, r := range text { repl, ok := s.charMap[r] @@ -1799,7 +1827,9 @@ func (s *serializer) mapSegments(text string) []mapSegment { }) run = i + utf8.RuneLen(r) } - return append(segs, mapSegment{text: s.normalized(text[run:])}) + segs = append(segs, mapSegment{text: s.normalized(text[run:])}) + s.segBuf = segs + return segs } // mapSegment is one unmapped run of text and, where the run ended at a diff --git a/xslt/serialize_streamalloc_test.go b/xslt/serialize_streamalloc_test.go new file mode 100644 index 00000000..c3f5ac34 --- /dev/null +++ b/xslt/serialize_streamalloc_test.go @@ -0,0 +1,94 @@ +package xslt + +import ( + "bytes" + "io" + goruntime "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// streamDoc is n copies of an element holding text, a comment and a PI, so +// that every path below has a node per copy to allocate for. +func streamDoc(t *testing.T, n int) xdm.Sequence { + t.Helper() + var b strings.Builder + b.WriteString(``) + for range n { + b.WriteString(`x ]]> y   z`) + } + b.WriteString(``) + tree, err := xdm.ParseString(b.String(), xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + return xdm.One(tree.Root) +} + +// The cdata-section-elements path built each section in a builder, a +// character map allocated a segment slice per text node, and comments and PIs +// were concatenated before being written. Each cost an allocation per node, +// so the count must not grow with the number of nodes. +func TestSerializeStreamAllocsFlat(t *testing.T) { + if raceEnabled { + t.Skip("allocation counts are not stable under -race") + } + for _, tc := range []struct { + name string + opts OutputSettings + cmap map[rune]string + }{ + {"cdata", OutputSettings{Method: "xml", CDataElements: []xdm.QName{{Local: "a"}}}, nil}, + {"charmap", OutputSettings{Method: "xml"}, map[rune]string{' ': " "}}, + {"comment-pi", OutputSettings{Method: "xml"}, nil}, + } { + count := func(seq xdm.Sequence) float64 { + return testing.AllocsPerRun(10, func() { + if err := Serialize(io.Discard, seq, tc.opts, tc.cmap); err != nil { + t.Fatal(err) + } + }) + } + few, many := count(streamDoc(t, 5)), count(streamDoc(t, 500)) + if many > few+5 { + t.Errorf("%s: 500 nodes %.0f allocations, 5 nodes %.0f; want the same", tc.name, many, few) + } + } +} + +// The text method concatenated the whole string value before writing it, a +// copy of the output and its builder's growth on top. It now writes each text +// node as it comes, so what it allocates must not grow with the output. +func TestSerializeTextMethodStreams(t *testing.T) { + if raceEnabled { + t.Skip("allocation counts are not stable under -race") + } + seq := streamDoc(t, 20000) + var out bytes.Buffer + if err := Serialize(&out, seq, OutputSettings{Method: "text"}, nil); err != nil { + t.Fatal(err) + } + if got := allocBytes(func() { + if err := Serialize(io.Discard, seq, OutputSettings{Method: "text"}, nil); err != nil { + t.Fatal(err) + } + }); got > uint64(out.Len())/4 { + t.Errorf("text method allocated %d bytes for %d bytes of output", got, out.Len()) + } +} + +// allocBytes is the heap f allocates, the least of three runs so that a +// collection landing inside one does not count against it. +func allocBytes(f func()) uint64 { + best := ^uint64(0) + for range 3 { + var a, b goruntime.MemStats + goruntime.ReadMemStats(&a) + f() + goruntime.ReadMemStats(&b) + best = min(best, b.TotalAlloc-a.TotalAlloc) + } + return best +} From 73af7a84f631fa8d4d99c1b085f5c1054b3a7378 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:16:24 -0400 Subject: [PATCH 13/98] xquery: resolve lifted operands without building a library Every evaluation of an expression with a lifted operand built an xpath.Library map and a closure per operand. liftedLib reads the operand index out of the xq-stepN name instead. XMark q1-q20 eval: 125.6 -> 119.4 MB. --- xquery/flwor_alloc_test.go | 9 ++++++++ xquery/operand.go | 45 +++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/xquery/flwor_alloc_test.go b/xquery/flwor_alloc_test.go index 20011127..3fa98f86 100644 --- a/xquery/flwor_alloc_test.go +++ b/xquery/flwor_alloc_test.go @@ -95,3 +95,12 @@ func TestFLWORResultReserved(t *testing.T) { t.Errorf("%d bytes, want at most 22,500,000", b) } } + +// TestLiftedOperandNoLibrary pins that evaluating an expression with a lifted +// operand does not build a function library each time. Measured: 6.75 MB +// before, 4.14 MB after. +func TestLiftedOperandNoLibrary(t *testing.T) { + if b := flworBytes(t, `for $a in 1 to 2000 return $a = 5`, 2000); b > 5_400_000 { + t.Errorf("%d bytes, want at most 5,400,000", b) + } +} diff --git a/xquery/operand.go b/xquery/operand.go index 1c7c32d4..c2e4a58f 100644 --- a/xquery/operand.go +++ b/xquery/operand.go @@ -479,23 +479,42 @@ func (n *operandExpr) sequence(ctx *evalContext) (xdm.Sequence, error) { func bindLifted(xp *xpath.Context, ops []liftedOperand, ctx *evalContext) *xpath.Context { - lib := xpath.NewLibrary(xp.Funcs) - for i, op := range ops { - item := op.n - lib.Add(xpath.Function{ - Name: xdm.QName{URI: callArgNS(ctx.sc), Local: stepFn(i)}, - Arity: 0, - Call: func(c *xpath.Context, _ []xdm.Sequence) (xdm.Sequence, error) { - return (&enclosed{items: []node{item}}).sequence( - &evalContext{xp: c, sc: ctx.sc, joins: ctx.joins}) - }, - }) - } sub := *xp - sub.Funcs = lib + sub.Funcs = &liftedLib{parent: xp.Funcs, uri: callArgNS(ctx.sc), ops: ops, ctx: ctx} return &sub } +// liftedLib is the library bindLifted installs: ops[i] as the zero-arity +// function stepFn(i), over parent. It reads the index out of the name rather +// than holding a map of the functions, which every evaluation had to build. +type liftedLib struct { + parent xpath.FunctionLibrary + uri string + ops []liftedOperand + ctx *evalContext +} + +func (l *liftedLib) Lookup(name xdm.QName, arity int) (xpath.Function, bool) { + if rest, ok := strings.CutPrefix(name.Local, "xq-step"); ok && arity == 0 && name.URI == l.uri { + if i, err := strconv.Atoi(rest); err == nil && i >= 0 && i < len(l.ops) && + rest == strconv.Itoa(i) { + item, ctx := l.ops[i].n, l.ctx + return xpath.Function{ + Name: name, + Arity: 0, + Call: func(c *xpath.Context, _ []xdm.Sequence) (xdm.Sequence, error) { + return (&enclosed{items: []node{item}}).sequence( + &evalContext{xp: c, sc: ctx.sc, joins: ctx.joins}) + }, + }, true + } + } + if l.parent != nil { + return l.parent.Lookup(name, arity) + } + return xpath.Function{}, false +} + func (n *operandExpr) eval(out *builderRef, ctx *evalContext) error { seq, err := n.sequence(ctx) if err != nil { From 490fc5a7de4f5fe5a74304d20eac6257ae67dd6f Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:16:48 -0400 Subject: [PATCH 14/98] xslt: intern compiled namespace maps, lists and resolvers by content SchXslt redeclares a prefix on each CEN svrl:failed-assert, so the compiled sheet kept ~980 copies of one map, binding list and resolvers. Intern them by content per Compile, one resolver per literal element: CEN compiled 9.14 -> 6.06 MB, DocBook 9.71 -> 8.76 MB. --- xslt/compile_instr.go | 11 ++++- xslt/sharedns_test.go | 46 +++++++++++++++++++++ xslt/stylesheet.go | 96 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 149 insertions(+), 4 deletions(-) diff --git a/xslt/compile_instr.go b/xslt/compile_instr.go index fbebe36d..b93839a8 100644 --- a/xslt/compile_instr.go +++ b/xslt/compile_instr.go @@ -317,7 +317,7 @@ func (c *compiler) compileLiteralElement(n *xdm.Node) (Instruction, error) { // xsl:stylesheet and uses it inside a literal element expects the result // to carry the declaration, and the difference is observable through the // namespace axis of the constructed tree. - scope := n.InScopeNamespaces() + scope := inScopeNamespacesShared(n) prefixes := make([]string, 0, len(scope)) for p := range scope { prefixes = append(prefixes, p) @@ -343,14 +343,21 @@ func (c *compiler) compileLiteralElement(n *xdm.Node) (Instruction, error) { } instr.namespaces = append(instr.namespaces, nsBinding{prefix: p, uri: uri}) } + instr.namespaces = internBindings(instr.namespaces) + instr.excludedNamespaces = internBindings(instr.excludedNamespaces) + // One resolver serves every attribute of the element. + var ns *nsResolver for a := range n.Attrs() { // xsl:-prefixed attributes on a literal element are directives, not // output attributes. if a.Name().URI == xdm.NSXSL { continue } - avt, err := compileAVT(a.Value(), newNSResolver(n, "")) + if ns == nil { + ns = newNSResolver(n, "") + } + avt, err := compileAVT(a.Value(), ns) if err != nil { return nil, fmt.Errorf("in attribute %s: %w", a.Name().Lexical(), err) } diff --git a/xslt/sharedns_test.go b/xslt/sharedns_test.go index 784346ee..b70509c4 100644 --- a/xslt/sharedns_test.go +++ b/xslt/sharedns_test.go @@ -54,3 +54,49 @@ func TestInScopeNamespacesShared(t *testing.T) { t.Error("maps shared outside a compilation") } } + +// SchXslt redeclares one prefix on each of the CEN rules' svrl:failed-assert +// elements. Elements that each declare the same bindings share one map, one +// resolver and one binding list in the compiled stylesheet, rather than a copy +// per element. +func TestNamespacesInternedByContent(t *testing.T) { + const sheet = ` + + + + +` + doc, err := xdm.ParseString(sheet, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var outs []*xdm.Node + for n := range doc.Root.Descendants() { + if n.Kind() == xdm.KindElement && n.Name().Local == "out" { + outs = append(outs, n) + } + } + setSharedNS(map[*xdm.Node]map[string]string{}) + ptr := func(m map[string]string) uintptr { return reflect.ValueOf(m).Pointer() } + if ptr(inScopeNamespacesShared(outs[0])) != ptr(inScopeNamespacesShared(outs[1])) { + t.Error("two elements declaring the same bindings do not share a map") + } + if newNSResolver(outs[0], "") != newNSResolver(outs[1], "") { + t.Error("two elements with the same static context do not share a resolver") + } + setSharedNS(nil) + + st, err := Compile(doc.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + var lits []*literalElemInstr + for _, in := range st.named["t"].Body { + if l, ok := in.(*literalElemInstr); ok { + lits = append(lits, l) + } + } + if len(lits) != 2 || len(lits[0].namespaces) == 0 || &lits[0].namespaces[0] != &lits[1].namespaces[0] { + t.Error("two literal elements with the same bindings do not share one list") + } +} diff --git a/xslt/stylesheet.go b/xslt/stylesheet.go index 1f23b2d6..b0a2b9ed 100644 --- a/xslt/stylesheet.go +++ b/xslt/stylesheet.go @@ -2,9 +2,11 @@ package xslt import ( "fmt" + "reflect" "sort" "strings" "sync" + "unsafe" "github.com/knroy/go-xml/v2/xdm" "github.com/knroy/go-xml/v2/xpath" @@ -700,17 +702,57 @@ func Compile(doc *xdm.Node, opts CompileOptions) (*Stylesheet, error) { // Outside a compilation m is nil and nothing is cached. It has its own mutex // rather than relying on compileMu because a running transform, which does // not hold compileMu, can reach inScopeNamespacesShared too. +// +// byContent and lists intern the maps and the literal elements' binding lists +// by content as well: SchXslt redeclares one prefix on each of the CEN +// rules' ~980 svrl:failed-assert elements, so keying by the declaring +// element alone kept that many identical copies in the compiled stylesheet. var sharedNS struct { sync.Mutex - m map[*xdm.Node]map[string]string + m map[*xdm.Node]map[string]string + byContent map[string]map[string]string + lists map[string][]nsBinding + resolvers map[nsResolverKey]*nsResolver } func setSharedNS(m map[*xdm.Node]map[string]string) { sharedNS.Lock() sharedNS.m = m + sharedNS.byContent, sharedNS.lists, sharedNS.resolvers = nil, nil, nil + if m != nil { + sharedNS.byContent = map[string]map[string]string{} + sharedNS.lists = map[string][]nsBinding{} + sharedNS.resolvers = map[nsResolverKey]*nsResolver{} + } sharedNS.Unlock() } +// internBindings returns a shared copy of s for the rest of the compilation, +// or s itself outside one. The result must not be modified. +func internBindings(s []nsBinding) []nsBinding { + if len(s) == 0 { + return s + } + var b strings.Builder + for _, ns := range s { + b.WriteString(ns.prefix) + b.WriteByte(0) + b.WriteString(ns.uri) + b.WriteByte(0) + } + sharedNS.Lock() + defer sharedNS.Unlock() + if sharedNS.lists == nil { + return s + } + k := b.String() + if v, ok := sharedNS.lists[k]; ok { + return v + } + sharedNS.lists[k] = s + return s +} + // forgetSharedNS drops the cached maps, and the version memo, for a caller // that has just changed a stylesheet tree's namespaces or parent links // during a compilation. @@ -741,6 +783,24 @@ func inScopeNamespacesShared(el *xdm.Node) map[string]string { m, ok := sharedNS.m[d] if !ok { m = d.InScopeNamespaces() + // NUL cannot occur in a prefix or a URI, so it separates them. + prefixes := make([]string, 0, len(m)) + for p := range m { + prefixes = append(prefixes, p) + } + sort.Strings(prefixes) + var b strings.Builder + for _, p := range prefixes { + b.WriteString(p) + b.WriteByte(0) + b.WriteString(m[p]) + b.WriteByte(0) + } + if v, ok := sharedNS.byContent[b.String()]; ok { + m = v + } else { + sharedNS.byContent[b.String()] = m + } sharedNS.m[d] = m } return m @@ -1233,7 +1293,7 @@ func newNSResolver(el *xdm.Node, defaultElementNS string) *nsResolver { if defaultElementNS == "" { defaultElementNS = xpathDefaultNamespaceAt(el) } - return &nsResolver{ + r := &nsResolver{ bindings: inScopeNamespacesShared(el), defaultNS: defaultElementNS, baseURI: el.BaseURI(), @@ -1245,6 +1305,38 @@ func newNSResolver(el *xdm.Node, defaultElementNS string) *nsResolver { xpathVersion: xpathVersionAt(el), xsltVersion: declaredXSLTVersion(el), } + return internResolver(r) +} + +// nsResolverKey is an nsResolver as a map key. The bindings map is keyed by +// identity, which inScopeNamespacesShared has already made content identity +// for the compilation; every other field must be listed here. +type nsResolverKey struct { + bindings unsafe.Pointer + defaultNS, baseURI, collate string + compat bool + schema *xsd.Schema + pkg int + xpathVersion xpath.Version + xsltVersion float64 +} + +// internResolver returns the compilation's shared resolver equal to r, or r +// itself outside a compilation. A resolver is never modified once built +// (xsl:evaluate copies one to change it), so equal ones can be one. +func internResolver(r *nsResolver) *nsResolver { + sharedNS.Lock() + defer sharedNS.Unlock() + if sharedNS.resolvers == nil { + return r + } + k := nsResolverKey{reflect.ValueOf(r.bindings).UnsafePointer(), r.defaultNS, r.baseURI, + r.collation, r.compat, r.schema, r.pkg, r.xpathVersion, r.xsltVersion} + if v, ok := sharedNS.resolvers[k]; ok { + return v + } + sharedNS.resolvers[k] = r + return r } // LookupSchemaDeclaration implements xpath.SchemaTypes. From e056467b9691fc38ce6471962250b6d8c4283f4b Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:16:59 -0400 Subject: [PATCH 15/98] xsd: reuse two NFA state sets in matchNames matchNames made a new map-backed state set for every child. It now swaps two, clearing the reused one. dtd BenchmarkValidate: 1,295,033 -> 691,383 B/op, 14,609 -> 8,112 allocs/op. --- xsd/nfa.go | 9 +++++++-- xsd/seqmatch_alloc_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 xsd/seqmatch_alloc_test.go diff --git a/xsd/nfa.go b/xsd/nfa.go index 7801bf21..61e9ac52 100644 --- a/xsd/nfa.go +++ b/xsd/nfa.go @@ -395,8 +395,13 @@ func matchNames(m *contentModel, match func(i, pos int) bool, n, budget int) (ok enter := make([]int, len(m.counters)) var vecs [][]int + // Two sets, swapped each step: a fresh map per child was three quarters + // of the DTD validator's allocation. + spare := newStateSet() for i := 0; i < n; i++ { - next := newStateSet() + next := spare + clear(next.seen) + next.order = next.order[:0] for _, st := range states.order { for _, pos := range candidates(m, st) { if !match(i, pos) { @@ -421,7 +426,7 @@ func matchNames(m *contentModel, match func(i, pos int) bool, n, budget int) (ok if next.len() > budget { return false, i, errMatchStates } - states = next + states, spare = next, states } for _, st := range states.order { if accepting(m, st, cur) { diff --git a/xsd/seqmatch_alloc_test.go b/xsd/seqmatch_alloc_test.go new file mode 100644 index 00000000..ca54dcac --- /dev/null +++ b/xsd/seqmatch_alloc_test.go @@ -0,0 +1,28 @@ +package xsd + +import ( + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// matchNames swaps two state sets instead of making one per child, which was +// three quarters of the DTD validator's allocation. Allocation per Match must +// not grow with the number of children. +func TestSequenceMatcherReusesStateSets(t *testing.T) { + p := &Particle{MinOccurs: 0, MaxOccurs: Unbounded, Term: &ElementDecl{Name: xdm.QName{Local: "a"}}} + m, err := NewSequenceMatcher(p) + if err != nil { + t.Fatal(err) + } + names := make([]xdm.QName, 1000) + for i := range names { + names[i] = xdm.QName{Local: "a"} + } + if ok, _ := m.Match(names); !ok { + t.Fatal("a* rejected a run of a") + } + if n := testing.AllocsPerRun(5, func() { m.Match(names) }); n > 100 { + t.Errorf("Match over 1000 children made %v allocations, want at most 100", n) + } +} From ec1ac54a08648d2d0e75cde3a569613075b56e9e Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:18:22 -0400 Subject: [PATCH 16/98] xdm: keep line starts, not the source text, for positions TrackPositions kept the whole document text for the tree's life only to count lines in. The tee now records where lines start as the text goes by. Tracked parse-10mb tree: 50.4 -> 41.5 MB retained. --- xdm/build.go | 8 +++--- xdm/node.go | 24 +++++++----------- xdm/parse.go | 55 ++++++++++++++++++++++++----------------- xdm/record.go | 51 ++++++++++++++++++++++++++++---------- xdm/retain_test.go | 12 ++++----- xdm/source_copy_test.go | 50 ++++++++++++++++++++++--------------- 6 files changed, 120 insertions(+), 80 deletions(-) diff --git a/xdm/build.go b/xdm/build.go index 559916f5..a1401791 100644 --- a/xdm/build.go +++ b/xdm/build.go @@ -261,10 +261,10 @@ func (t *Tree) CopySourceFrom(src *Tree) { } t.CopyDTDFrom(src) t.XMLVersion = src.XMLVersion - if s := src.srcText(); s != "" { - t.ownSource().src = s + if l := src.lineIdx(); l != nil { + t.ownSource().lines = l } else if t.source != nil { - t.source.src = "" + t.source.lines = nil } } @@ -279,7 +279,7 @@ func CopyPosition(dst, src *Node) { if !ok { return } - if _, foreign := src.tree.foreign(src.self); !foreign && dst.tree.srcText() == src.tree.srcText() { + if _, foreign := src.tree.foreign(src.self); !foreign && dst.tree.lineIdx() == src.tree.lineIdx() { dst.setOffset(src.offset()) return } diff --git a/xdm/node.go b/xdm/node.go index 4d0384d1..3f0fb209 100644 --- a/xdm/node.go +++ b/xdm/node.go @@ -4,7 +4,7 @@ import ( "errors" "math" "math/big" - "sort" + "slices" "strconv" "strings" @@ -598,7 +598,7 @@ func (n *Node) FirstElement(uri, local string) *Node { // // A caller that caches trees needs it: one parsed without positions cannot // serve a request that needs them, and the only way to tell is to ask. -func (t *Tree) HasPositions() bool { return t != nil && t.srcText() != "" } +func (t *Tree) HasPositions() bool { return t != nil && t.lineIdx() != nil } // positionAt converts a byte offset into a 1-based line and column. // @@ -606,25 +606,19 @@ func (t *Tree) HasPositions() bool { return t != nil && t.srcText() != "" } // decoder, and a caller pointing an editor at the failure wants the same // units the decoder used. func (t *Tree) positionAt(off int) (line, col int, ok bool) { - s := t.source - if s == nil || s.src == "" || off < 0 || off > len(s.src) { + l := t.lineIdx() + if l == nil || off < 0 || off > l.size { return 0, 0, false } - s.lineOnce.Do(func() { - // Line 1 starts at offset 0; every byte after a newline starts another. - s.lineStarts = append(s.lineStarts, 0) - for i := 0; i < len(s.src); i++ { - if s.src[i] == '\n' { - s.lineStarts = append(s.lineStarts, i+1) - } - } - }) // The line is the last one starting at or before off. - i := sort.SearchInts(s.lineStarts, off+1) - 1 + i, found := slices.BinarySearch(l.starts, int32(off)) + if !found { + i-- + } if i < 0 { return 0, 0, false } - return i + 1, off - s.lineStarts[i] + 1, true + return i + 1, off - int(l.starts[i]) + 1, true } // The six functions below are the name-only face of the type derivation diff --git a/xdm/parse.go b/xdm/parse.go index 83162da6..57bbb994 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -175,7 +175,7 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { // that costs more than accepting. Wrapped here the ReadAll hits the // limited reader and stops at the bound. // A reader that knows its length (strings.Reader, bytes.Reader) lets the - // position-tracking copy below be sized once instead of grown by doubling. + // read windows and the tokeniser's buffers be sized to it. sizeHint := 0 if l, ok := r.(interface{ Len() int }); ok { sizeHint = l.Len() @@ -223,27 +223,31 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { trackPos := opts.TrackPositions var srcBuf strings.Builder - // The source is kept when positions are tracked, and also when a DOCTYPE - // is permitted: an entity whose replacement text holds markup forces a - // re-parse of the substituted source, and by the time that is known the - // reader is partly consumed and the decoder has buffered ahead into it. + // The source is copied when a DOCTYPE is permitted: an entity whose + // replacement text holds markup forces a re-parse of the substituted + // source, and by the time that is known the reader is partly consumed and + // the decoder has buffered ahead into it. // // The copy is needed only until the document element opens: a re-parse // can start only at the DOCTYPE, which must come first, so past that point - // the tee stops and its copy is dropped unless positions are tracked. - // entitiesExpanded marks the second parse, which has no entities left to - // find and so needs no copy at all. - // Only a tracked copy is kept to the end, so only it is sized up front; - // never past MaxBytes, which the reader will refuse to go beyond. + // the tee stops copying and the copy is dropped. entitiesExpanded marks + // the second parse, which has no entities left to find and so needs no + // copy at all. + // + // Positions need only where the source's lines start, which the tee + // records as the text goes by. if !verbatim(whole) { whole = "" } - if whole == "" && trackPos && sizeHint > 0 && (maxBytes <= 0 || int64(sizeHint) <= maxBytes) { - srcBuf.Grow(sizeHint) - } var tee *srcTee if whole == "" && (trackPos || (opts.AllowDOCTYPE && !opts.entitiesExpanded)) { - tee = &srcTee{r: r, buf: &srcBuf} + tee = &srcTee{r: r} + if opts.AllowDOCTYPE && !opts.entitiesExpanded { + tee.buf = &srcBuf + } + if trackPos { + tee.lines = &lineIndex{} + } r = tee } @@ -381,7 +385,7 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { // kept for one is dropped; and with no entity table installed // by now none ever will be, so the charge reader has nothing // to buffer for. - if tee != nil && !trackPos { + if tee != nil { tee.buf = nil srcBuf = strings.Builder{} } @@ -712,13 +716,15 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { if trackPos { // The decoder stops reading at the end of the root element, so the - // tee holds everything up to there — which is all any offset can + // tee has seen everything up to there — which is all any offset can // point into. - src := srcBuf.String() + lines := &lineIndex{} if whole != "" { - src = whole + lines.add(unsafe.Slice(unsafe.StringData(whole), len(whole))) + } else { + lines = tee.lines } - tree.ownSource().src = src + tree.ownSource().lines = lines } tree.XMLVersion = "1.0" if dec.IsVersion11() { @@ -1212,10 +1218,12 @@ func charsetReader(charset string, input io.Reader) (io.Reader, error) { return nil, fmt.Errorf("unsupported encoding %q", charset) } -// srcTee copies what is read from r into buf until buf is set to nil. +// srcTee copies what is read from r into buf until buf is set to nil, and +// indexes its lines in lines when that is set. type srcTee struct { - r io.Reader - buf *strings.Builder + r io.Reader + buf *strings.Builder + lines *lineIndex } func (t *srcTee) Read(p []byte) (int, error) { @@ -1223,6 +1231,9 @@ func (t *srcTee) Read(p []byte) (int, error) { if t.buf != nil { t.buf.Write(p[:n]) } + if t.lines != nil { + t.lines.add(p[:n]) + } return n, err } diff --git a/xdm/record.go b/xdm/record.go index c583c3c0..f2a59ddf 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1,7 +1,9 @@ package xdm import ( + "bytes" "iter" + "math" "math/bits" "sort" "strconv" @@ -161,15 +163,10 @@ type treeSource struct { // // Empty unless ParseOptions.ExternalEntities permitted the read. externalSubset string - // src is the document text, retained only when the caller asks for - // positions. It is what makes Position able to count lines. - src string - // lineStarts holds the byte offset of each line, built on first use. - // Resolving a position is then a binary search rather than a scan from - // the start of the document, which matters when a validator reports - // thousands of failures over one large file. - lineStarts []int - lineOnce sync.Once + // lines is where the document's lines start, kept only when the caller + // asks for positions. It is what makes Position able to count lines; the + // text itself was kept for that once, at a byte per input byte. + lines *lineIndex offsets [][]int32 // by chunk: source offset + 1, made on first use foreignPos map[uint32][2]int32 // positions copied from another document } @@ -182,12 +179,40 @@ func (t *Tree) ownSource() *treeSource { return t.source } -// srcText is the retained document text, or "". -func (t *Tree) srcText() string { +// lineIdx is the document's line index, or nil. +func (t *Tree) lineIdx() *lineIndex { if t.source == nil { - return "" + return nil + } + return t.source.lines +} + +// lineIndex holds the offset each line of a source text starts at, so that +// resolving a position is a binary search rather than a scan from the start +// of the document, which matters when a validator reports thousands of +// failures over one large file. Offsets past math.MaxInt32 are not recorded: +// no node offset (encodeOffset) points there. +type lineIndex struct { + starts []int32 // line 1 starts at 0; every byte after a newline starts another + size int // how many bytes of text were indexed +} + +// add indexes the next len(b) bytes of the text. +func (l *lineIndex) add(b []byte) { + if l.starts == nil { + l.starts = []int32{0} + } + for i := 0; ; { + j := bytes.IndexByte(b[i:], '\n') + if j < 0 { + break + } + i += j + 1 + if off := l.size + i; off < math.MaxInt32 { + l.starts = append(l.starts, int32(off)) + } } - return t.source.src + l.size += len(b) } // extSubset is the external subset text, or "". diff --git a/xdm/retain_test.go b/xdm/retain_test.go index d33e6655..079e89b2 100644 --- a/xdm/retain_test.go +++ b/xdm/retain_test.go @@ -6,16 +6,16 @@ import ( "testing" ) -// retainedPerTree is the heap still live, per tree, with n parses of doc -// kept, after a collection. -func retainedPerTree(t *testing.T, doc string, opts ParseOptions, n int) int64 { +// retainedPerTree is the heap still live, per tree, with the trees of n +// calls of parse kept, after a collection. +func retainedPerTree(t *testing.T, n int, parse func() (*Tree, error)) int64 { t.Helper() trees := make([]*Tree, n) var m0, m1 runtime.MemStats runtime.GC() runtime.ReadMemStats(&m0) for i := range trees { - tr, err := ParseString(doc, opts) + tr, err := parse() if err != nil { t.Fatal(err) } @@ -45,8 +45,8 @@ func TestNamespaceURIDoesNotPinArena(t *testing.T) { withNS := `` without := `` const n = 200 - plain := retainedPerTree(t, without, ParseOptions{}, n) - ns := retainedPerTree(t, withNS, ParseOptions{}, n) + plain := retainedPerTree(t, n, func() (*Tree, error) { return ParseString(without, ParseOptions{}) }) + ns := retainedPerTree(t, n, func() (*Tree, error) { return ParseString(withNS, ParseOptions{}) }) t.Logf("retained per tree: %d bytes without a declaration, %d with", plain, ns) if ns-plain > 2<<10 { t.Errorf("a namespace declaration costs %d bytes per kept tree, want under 2 KB", ns-plain) diff --git a/xdm/source_copy_test.go b/xdm/source_copy_test.go index b35bcc73..ac26be58 100644 --- a/xdm/source_copy_test.go +++ b/xdm/source_copy_test.go @@ -97,31 +97,41 @@ func TestSourceCopyStillServesItsUsers(t *testing.T) { }) } -// TrackPositions keeps the whole source to count lines in. A reader that -// knows its length lets that copy be allocated once at its final size, -// rather than grown by doubling to about twice the document in all. -func TestTrackedCopyIsSizedOnce(t *testing.T) { - doc := "\n" + bigBody(100000) + "\n" - parse := func(track bool) func() { - return func() { - if _, err := ParseString(doc, ParseOptions{TrackPositions: track, MaxBytes: -1, MaxNodes: -1}); err != nil { - t.Fatal(err) - } +// TrackPositions kept the whole source text, a byte per input byte for the +// life of the tree, to count lines in. It keeps where the lines start now: +// four bytes a line. Measured on parse-10mb, a tracked tree went from 50.4 MB +// to 41.5 MB. +func TestTrackedTreeKeepsNoSourceText(t *testing.T) { + if raceEnabled { + t.Skip("heap sizes are not stable under -race") + } + doc := "\n" + bigBody(20000) + "\n" + parse := func(track bool) func() (*Tree, error) { + return func() (*Tree, error) { + // A reader, not ParseString, whose string the tree may share. + return Parse(strings.NewReader(doc), ParseOptions{TrackPositions: track, MaxBytes: -1, MaxNodes: -1}) } } - plain := allocated(parse(false)) - tracked := allocated(parse(true)) - extra := int64(tracked) - int64(plain) - // Besides the copy, tracking keeps each node's source offset: four bytes - // a record, in a column a parse without positions does not have. - tr, err := ParseString(doc, ParseOptions{MaxBytes: -1, MaxNodes: -1}) + plain := retainedPerTree(t, 4, parse(false)) + tracked := retainedPerTree(t, 4, parse(true)) + tr, err := parse(true)() if err != nil { t.Fatal(err) } - column := 4 * int64(tr.n) - t.Logf("tracked copy cost %d bytes for a %d-byte document (offset column %d)", extra, len(doc), column) - if extra > int64(len(doc))*5/4+column { - t.Errorf("TrackPositions allocated %d bytes more than without (document %d bytes), want about one copy", + // Tracking keeps each element's source offset, four bytes a record in a + // column a parse without positions does not have, and the line starts. + column := int64(0) + for _, ch := range tr.chunks { + column += 4 * int64(cap(ch)) + } + lines := 8 * int64(strings.Count(doc, "\n")+1) // a slice grown by doubling + extra := tracked - plain + t.Logf("tracking kept %d bytes for a %d-byte document (offset column %d, line starts %d)", extra, len(doc), column, lines) + if extra > column+lines+int64(len(doc))/8 { + t.Errorf("a tracked tree kept %d bytes more than an untracked one (document %d bytes), want no copy of the text", extra, len(doc)) } + if line, col, ok := kids(kids(tr.Root)[0])[2*20000-1].Position(); !ok || line != 20001 || col != 1 { + t.Errorf("last element at line %d col %d (ok=%v), want line 20001 col 1", line, col, ok) + } } From f09c1a39c010b61ad2c57583287a279130d5c013 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:18:48 -0400 Subject: [PATCH 17/98] xsd: record each ID value in one map idOwners and ids were two maps with the same keys. They are now one map of {owner, n}. BenchmarkValidateInstance: 213,906 -> 183,403 B/op; 20,000 IDs: 3.50 MB -> 2.63 MB allocated. --- xsd/id_map_alloc_test.go | 43 +++++++++++++++++++++++++++++++++++ xsd/validate.go | 13 ++++------- xsd/validate_fastpath_test.go | 2 +- xsd/validate_simple.go | 28 +++++++++++++++-------- 4 files changed, 66 insertions(+), 20 deletions(-) create mode 100644 xsd/id_map_alloc_test.go diff --git a/xsd/id_map_alloc_test.go b/xsd/id_map_alloc_test.go new file mode 100644 index 00000000..25bbde47 --- /dev/null +++ b/xsd/id_map_alloc_test.go @@ -0,0 +1,43 @@ +package xsd + +import ( + "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// Each ID value is recorded in one map. It was two with the same keys, an +// owner map and a count map, which on a document of nothing but IDs was most +// of what validation allocated: 20,000 IDs took 3.50 MB with two maps and +// 2.63 MB with one. Map growth is deterministic, so the bound sits between. +func TestIDValuesRecordedOnce(t *testing.T) { + s := mustParseSchema(t, ` + + + + + + `) + var b strings.Builder + b.WriteString("") + for i := 0; i < 20000; i++ { + b.WriteString(``) + } + b.WriteString("") + tree, err := xdm.ParseString(b.String(), xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var m0, m1 runtime.MemStats + runtime.ReadMemStats(&m0) + err = s.Validate(tree.Root, ValidateOptions{}) + runtime.ReadMemStats(&m1) + if err != nil { + t.Fatal(err) + } + if got := m1.TotalAlloc - m0.TotalAlloc; got > 3_000_000 { + t.Errorf("validating 20,000 IDs allocated %d bytes, want at most 3,000,000", got) + } +} diff --git a/xsd/validate.go b/xsd/validate.go index 33246d9d..49fbee8b 100644 --- a/xsd/validate.go +++ b/xsd/validate.go @@ -174,7 +174,7 @@ func (s *Schema) newValidator(ctx context.Context, opts ValidateOptions) *valida if opts.MaxDepth == 0 { opts.MaxDepth = DefaultMaxDepth } - return &validator{ctx: ctx, schema: s, opts: opts, ids: map[string]int{}} + return &validator{ctx: ctx, schema: s, opts: opts, ids: map[string]idDef{}} } // run assesses root, which is a document or an element. @@ -356,17 +356,12 @@ type validator struct { // typed copy of a document node. stripIgnorable bool - // ids records every xs:ID value seen and every xs:IDREF, so that - // Validation Root Valid (ID/IDREF) can be checked once at the end. A - // count of zero means the value was referenced but never defined. - ids map[string]int + // ids records every xs:ID value seen, so that Validation Root Valid + // (ID/IDREF) can be checked once at the end. + ids map[string]idDef // idrefs are the referenced values, checked against ids at the end. idrefs []idref - // idOwners maps each ID value to the element that defined it, so that - // the same value on two attributes of one element counts once. XSD 1.1 - // permits an element to carry several ID attributes. - idOwners map[string]*xdm.Node // dupIDs lists each ID value in the order its second definition was met, // which is where a streaming validator reports it; checkIDs reports in // this order so the errors (and the one MaxErrors keeps) are stable. diff --git a/xsd/validate_fastpath_test.go b/xsd/validate_fastpath_test.go index 454af7f3..bc17de99 100644 --- a/xsd/validate_fastpath_test.go +++ b/xsd/validate_fastpath_test.go @@ -127,7 +127,7 @@ func TestIdentityBookkeepingOnlyInsideAScope(t *testing.T) { root := tree.Root.ChildElements()[0] in := root.ChildElements()[2] v := &validator{schema: s, opts: ValidateOptions{MaxErrors: DefaultMaxErrors, - MaxDepth: DefaultMaxDepth}, ids: map[string]int{}} + MaxDepth: DefaultMaxDepth}, ids: map[string]idDef{}} v.validateElement(root, s.Elements[xdm.QName{Local: "root"}]) // 2 and 02 are one xs:int, so the unique is violated: the scope's diff --git a/xsd/validate_simple.go b/xsd/validate_simple.go index 1d0aab04..6d578d04 100644 --- a/xsd/validate_simple.go +++ b/xsd/validate_simple.go @@ -1046,19 +1046,27 @@ func (v *validator) recordID(owner *xdm.Node, value string) { // Nothing to denote; see idOwner. return } - if v.idOwners == nil { - v.idOwners = map[string]*xdm.Node{} - } - if prev, seen := v.idOwners[value]; seen { - if prev != owner { - if v.ids[value]++; v.ids[value] == 2 { + if d, seen := v.ids[value]; seen { + if d.owner != owner { + d.n++ + v.ids[value] = d + if d.n == 2 { v.dupIDs = append(v.dupIDs, value) } } return } - v.idOwners[value] = owner - v.ids[value]++ + v.ids[value] = idDef{owner: owner, n: 1} +} + +// idDef is one ID value's definitions. It was two maps with the same keys. +type idDef struct { + // owner is the element that first defined the value, so that the same + // value on two attributes of one element counts once. XSD 1.1 permits + // an element to carry several ID attributes. + owner *xdm.Node + // n is the number of elements defining the value. + n int } // idKind reports whether a type is or derives from xs:ID, xs:IDREF or @@ -1118,10 +1126,10 @@ func idKind(t *SimpleType, value string) string { func (v *validator) checkIDs() { for _, value := range v.dupIDs { v.fail(nil, "cvc-id.2", - "ID value %q is defined %d times", value, v.ids[value]) + "ID value %q is defined %d times", value, v.ids[value].n) } for _, ref := range v.idrefs { - if v.ids[ref.value] == 0 { + if v.ids[ref.value].n == 0 { v.fail(ref.node, "cvc-id.1", "IDREF %q does not match any ID in the document", ref.value) } From f14824737ebb1fd2d33ebc676efa8e100bbbab1a Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:19:12 -0400 Subject: [PATCH 18/98] xpath: append a predicate-free forward step straight onto the path result evalStepOver built a selection slice per input node and then copied it into the result. A forward step with no predicates now appends onto the result, which the final sort orders and deduplicates. XMark q15 eval: 345.7 -> 244.9 KB. --- xpath/eval.go | 44 ++++++++++++++++++++++++++--------------- xpath/stepfocus_test.go | 14 +++++++++++++ 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/xpath/eval.go b/xpath/eval.go index 1997eaf4..bb57c673 100644 --- a/xpath/eval.go +++ b/xpath/eval.go @@ -74,22 +74,7 @@ func (e *Step) Eval(ctx *Context) (xdm.Sequence, error) { // nothing would read. func (e *Step) evalFrom(ctx *Context, node *xdm.Node) (xdm.Sequence, error) { var err error - principal := e.Axis.PrincipalKind() - var selected xdm.Sequence - if nt, ok := e.Test.(*NameTest); ok && (e.Axis == AxisDescendant || e.Axis == AxisDescendantOrSelf) { - // The direct walk selects what walkAxis would, in the same order. - if e.Axis == AxisDescendantOrSelf && nt.Matches(node, principal) { - selected = append(selected, node) - } - selected = appendNamedDescendants(selected, node, nt) - } else { - walkAxis(node, e.Axis, func(n *xdm.Node) bool { - if e.Test.Matches(n, principal) { - selected = append(selected, n) - } - return true - }) - } + selected := e.appendFrom(nil, node) // Predicates apply in axis order, so a reverse axis numbers positions // from the context node outwards. @@ -105,6 +90,26 @@ func (e *Step) evalFrom(ctx *Context, node *xdm.Node) (xdm.Sequence, error) { return xdm.SortDocumentOrder(selected), nil } +// appendFrom appends to selected the nodes e's axis and node test select +// from node, in axis order, before any predicate. +func (e *Step) appendFrom(selected xdm.Sequence, node *xdm.Node) xdm.Sequence { + principal := e.Axis.PrincipalKind() + if nt, ok := e.Test.(*NameTest); ok && (e.Axis == AxisDescendant || e.Axis == AxisDescendantOrSelf) { + // The direct walk selects what walkAxis would, in the same order. + if e.Axis == AxisDescendantOrSelf && nt.Matches(node, principal) { + selected = append(selected, node) + } + return appendNamedDescendants(selected, node, nt) + } + walkAxis(node, e.Axis, func(n *xdm.Node) bool { + if e.Test.Matches(n, principal) { + selected = append(selected, n) + } + return true + }) + return selected +} + // Eval implements Expr for a path expression. func (e *PathExpr) Eval(ctx *Context) (xdm.Sequence, error) { var cur xdm.Sequence @@ -234,6 +239,13 @@ func evalStepOver(ctx *Context, input xdm.Sequence, step Expr, last bool) (xdm.S var v xdm.Sequence var err error st, fresh := step.(*Step) + if fresh && len(st.Predicates) == 0 && !st.Axis.IsReverse() { + // Nothing numbers this node's selection apart from the others, + // so it goes straight onto out; the sort below orders and + // deduplicates the whole, as it would the per-node results. + out = st.appendFrom(out, it.(*xdm.Node)) + continue + } if fresh { v, err = st.evalFrom(ctx, it.(*xdm.Node)) } else if da, ok := step.(*descendantAttrs); ok { diff --git a/xpath/stepfocus_test.go b/xpath/stepfocus_test.go index 5e2d52dc..e8463c39 100644 --- a/xpath/stepfocus_test.go +++ b/xpath/stepfocus_test.go @@ -78,6 +78,20 @@ func TestPathStepWithoutFocusCopy(t *testing.T) { if perNode > 4.5 { t.Errorf("a child step allocated %.2f times per input node, want at most 4.5", perNode) } + + // A predicate-free forward step appends what it selects straight onto + // the result, rather than through a slice per input node: "/r/a/b" over + // the 200 nodes measured 219 allocations with the slice and 20 without. + path := MustCompile("/r/a/b", nil) + whole := testing.AllocsPerRun(20, func() { + if _, err := path.Eval(ctx); err != nil { + t.Fatal(err) + } + }) + t.Logf("allocations for /r/a/b: %.0f", whole) + if whole > 100 { + t.Errorf("/r/a/b allocated %.0f times, want at most 100", whole) + } } // evalStepOver keeps the first axis step's result as its accumulator instead From 70ff9808d50807dc734099fc457c4fc622aa2ed0 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:20:18 -0400 Subject: [PATCH 19/98] xpath: step an int64-bounded range without big arithmetic evalRange made a big.Int and a big.Rat per value, and NewIntegerFromRat then narrowed each back to int64. Bounds that fit in int64 now loop as int64. distinct-values(1 to 100000) benchmark: 96.0 -> 76.8 MB/op. --- xpath/limits_test.go | 24 ++++++++++++++++++++++++ xpath/operators.go | 11 +++++++++++ xpath/saxon_diff_test.go | 7 +++++++ 3 files changed, 42 insertions(+) diff --git a/xpath/limits_test.go b/xpath/limits_test.go index 427488bc..00d0228f 100644 --- a/xpath/limits_test.go +++ b/xpath/limits_test.go @@ -320,3 +320,27 @@ func TestAdoptBudgetFromNilIsAFreshRoot(t *testing.T) { "budget, leaving the evaluation unbounded") } } + +// TestRangeInt64Allocations pins that a range whose bounds fit in int64 makes +// each value with NewInteger rather than through a big.Int and a big.Rat. +// "1000 to 2999" measured 16,010 allocations through big arithmetic and 8,009 +// without. +func TestRangeInt64Allocations(t *testing.T) { + c := MustCompile(`1000 to 2999`, nil) + ctx := NewContext(nil, Builtins()) + var n int + allocs := testing.AllocsPerRun(5, func() { + v, err := c.Eval(ctx) + if err != nil { + t.Fatal(err) + } + n = len(v) + }) + if n != 2000 { + t.Fatalf("%d items, want 2000", n) + } + t.Logf("%.0f allocations", allocs) + if allocs > 12000 { + t.Errorf("%.0f allocations, want at most 12,000", allocs) + } +} diff --git a/xpath/operators.go b/xpath/operators.go index ab98cd06..bb7937d7 100644 --- a/xpath/operators.go +++ b/xpath/operators.go @@ -862,6 +862,17 @@ func (e *BinaryOp) evalRange(ctx *Context) (xdm.Sequence, error) { return nil, err } out := make(xdm.Sequence, 0, count) + // Bounds that fit in int64 are stepped as one, which NewIntegerFromRat + // would narrow each value to anyway. Testing v against the end before + // incrementing it keeps a range ending at MaxInt64 from overflowing. + if lo.IsInt64() && hi.IsInt64() { + for v, end := lo.Int64(), hi.Int64(); ; v++ { + out = append(out, xdm.NewInteger(v)) + if v == end { + return out, nil + } + } + } // Stepping a big.Int keeps the arbitrary precision through the whole // range: the values are what the expression named, not a wrapped // approximation of them. diff --git a/xpath/saxon_diff_test.go b/xpath/saxon_diff_test.go index b20ed4dc..06160a41 100644 --- a/xpath/saxon_diff_test.go +++ b/xpath/saxon_diff_test.go @@ -515,6 +515,13 @@ func TestRangeBoundBeyondInt64(t *testing.T) { {`count(1000000000000000000000 to 1000000000000000000003)`, "4"}, {`(1000000000000000000000 to 1000000000000000000003)[2]`, "1000000000000000000001"}, {`count(18446744073709551616 to 18446744073709551620)`, "5"}, + // Bounds inside int64 are stepped as int64, up to the last value. + {`string-join(for $i in 9223372036854775806 to 9223372036854775807 return string($i), ",")`, + "9223372036854775806,9223372036854775807"}, + {`string-join(for $i in -9223372036854775808 to -9223372036854775807 return string($i), ",")`, + "-9223372036854775808,-9223372036854775807"}, + {`string-join(for $i in 9223372036854775807 to 9223372036854775808 return string($i), ",")`, + "9223372036854775807,9223372036854775808"}, // A range whose length is genuinely unbounded is still refused, and // for the length rather than for the size of the numbers. {`count(-1000000000000000000003 to -1000000000000000000000)`, "4"}, From dcb15d1954a1754d16d571aedec6180ef3db3121 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:20:41 -0400 Subject: [PATCH 20/98] xslt: keep namespace maps, not elements, in compiled instructions xsl:element, xsl:attribute and xsl:message/@error-code kept their stylesheet element only for LookupPrefix, which pinned every imported module tree. They keep the shared in-scope map instead: DocBook compiled (own resolver) 12.36 -> 9.31 MB. --- xslt/compile_instr.go | 6 ++-- xslt/instructions.go | 6 ++-- xslt/moduletree_test.go | 71 +++++++++++++++++++++++++++++++++++++++++ xslt/stylesheet.go | 17 +++++++++- 4 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 xslt/moduletree_test.go diff --git a/xslt/compile_instr.go b/xslt/compile_instr.go index b93839a8..454b9a61 100644 --- a/xslt/compile_instr.go +++ b/xslt/compile_instr.go @@ -1023,7 +1023,7 @@ func (c *compiler) compileElement(n *xdm.Node, ns xpath.NamespaceResolver) (Inst if err != nil { return nil, err } - instr := &elementInstr{name: nameAVT, scope: n, attrSets: sets, + instr := &elementInstr{name: nameAVT, scope: inScopeNamespacesShared(n), attrSets: sets, baseURI: n.BaseURI(), noInherit: noAttr(n, "inherit-namespaces")} if instr.validation, err = compileValidation(n, ""); err != nil { return nil, err @@ -1051,7 +1051,7 @@ func (c *compiler) compileAttribute(n *xdm.Node, ns xpath.NamespaceResolver) (In if err != nil { return nil, err } - instr := &attributeInstr{name: nameAVT, scope: n} + instr := &attributeInstr{name: nameAVT, scope: inScopeNamespacesShared(n)} if sepAttr := n.Attr("", "separator"); sepAttr != nil { sep, err := compileAVT(sepAttr.Value(), ns) if err != nil { @@ -1148,7 +1148,7 @@ func (c *compiler) compileMessage(n *xdm.Node, ns xpath.NamespaceResolver) (Inst if err != nil { return nil, err } - instr.errorCode, instr.errorCodeNS = avt, n + instr.errorCode, instr.errorCodeNS = avt, inScopeNamespacesShared(n) } if v := n.AttrValue("terminate"); v != "" { avt, err := compileAVT(v, ns) diff --git a/xslt/instructions.go b/xslt/instructions.go index 28c96a51..af05d7e6 100644 --- a/xslt/instructions.go +++ b/xslt/instructions.go @@ -1265,7 +1265,7 @@ type elementInstr struct { // the constructed element inherits. baseURI string namespace *avt - scope *xdm.Node + scope nsMap attrSets []attrSetRef body []Instruction // validation is the validation or type attribute, which asks for the @@ -1394,7 +1394,7 @@ type attributeInstr struct { separator *avt hasSeparator bool sel *xpath.Compiled - scope *xdm.Node + scope nsMap body []Instruction // validation carries [xsl:]validation and [xsl:]type, which assess the // constructed attribute exactly as they assess a constructed element. @@ -1697,7 +1697,7 @@ type messageInstr struct { // errorCodeNS resolves the prefix of an error-code that was written as a // lexical QName. The attribute is an AVT, so the name is not known until // run time and the namespace context has to be carried to meet it. - errorCodeNS *xdm.Node + errorCodeNS nsMap // xslt30 enables the 3.0 readings: the wider @terminate vocabulary, and // surviving a dynamic error raised while building the content. xslt30 bool diff --git a/xslt/moduletree_test.go b/xslt/moduletree_test.go new file mode 100644 index 00000000..394c4e7c --- /dev/null +++ b/xslt/moduletree_test.go @@ -0,0 +1,71 @@ +package xslt + +import ( + "context" + goruntime "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// moduleWithPayload serves one imported module carrying a 4 MB attribute the +// compiler ignores, so that whether the compiled stylesheet keeps the module's +// tree alive shows in the retained heap. +type moduleWithPayload struct{} + +func (moduleWithPayload) ResolveModule(href, base string) (*xdm.Node, string, error) { + mod := ` + + + + +` + d, err := xdm.ParseString(mod, xdm.ParseOptions{}) + if err != nil { + return nil, "", err + } + return d.Root, "urn:mod", nil +} + +func heapAlloc() uint64 { + var m goruntime.MemStats + goruntime.GC() + goruntime.GC() + goruntime.ReadMemStats(&m) + return m.HeapAlloc +} + +// A compiled stylesheet does not keep an imported module's tree alive: +// xsl:element, xsl:attribute and xsl:message/@error-code keep the in-scope +// namespaces they resolve computed names against, not the element. +func TestCompiledSheetReleasesModuleTree(t *testing.T) { + const main = ` + + +` + doc, err := xdm.ParseString(main, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + before := heapAlloc() + st, err := Compile(doc.Root, CompileOptions{Resolver: moduleWithPayload{}}) + if err != nil { + t.Fatal(err) + } + if kept := int64(heapAlloc()) - int64(before); kept > 2<<20 { + t.Errorf("compiled stylesheet retains %d bytes; the 4 MB module tree is still reachable", kept) + } + res, err := st.Transform(context.Background(), nil, TransformOptions{InitialTemplate: "t"}) + if err != nil { + t.Fatal(err) + } + var b strings.Builder + if err := res.Serialize(&b); err != nil { + t.Fatal(err) + } + if got := b.String(); got != `` { + t.Errorf("output %q", got) + } +} diff --git a/xslt/stylesheet.go b/xslt/stylesheet.go index b0a2b9ed..d12606e2 100644 --- a/xslt/stylesheet.go +++ b/xslt/stylesheet.go @@ -1758,13 +1758,28 @@ func (r *nsResolver) ResolvePrefix(p string) (string, bool) { func (r *nsResolver) DefaultElementNamespace() string { return r.defaultNS } func (r *nsResolver) DefaultFunctionNamespace() string { return xdm.NSFN } +// prefixLookup is an element, or the in-scope namespaces a compiled +// instruction keeps in its place so as not to keep the stylesheet tree alive. +type prefixLookup interface { + LookupPrefix(prefix string) (string, bool) +} + +// nsMap is an element's in-scope namespaces, as InScopeNamespaces returns +// them (xml included), answering LookupPrefix exactly as the element would. +type nsMap map[string]string + +func (m nsMap) LookupPrefix(prefix string) (string, bool) { + uri, ok := m[prefix] + return uri, ok || prefix == "" +} + // resolveQNameAttr resolves a lexical QName found in a stylesheet attribute // against the element's namespace context. // // An unprefixed QName in an attribute value is in *no* namespace, unlike an // unprefixed element name test. This is the rule that makes // name="foo" and mean different things inside a default namespace. -func resolveQNameAttr(el *xdm.Node, lex string) (xdm.QName, error) { +func resolveQNameAttr(el prefixLookup, lex string) (xdm.QName, error) { lex = strings.TrimSpace(lex) // The EQName form Q{uri}local carries its own namespace, so no prefix has // to be in scope for it. The suite writes it wherever a QName is From 2ad3a32a8dda688eddac64a412530496101d1b0c Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:21:01 -0400 Subject: [PATCH 21/98] xdm: hand a long text's tokeniser buffer to the text store A text value of 8 KB or more was built in the decoder's buffer and copied into a block of its own. The buffer becomes the block when it is at most 1/8 larger than the value. E-invoice corpus: 4.10 -> 3.49 MB allocated a pass. --- internal/xmltok/xmltok.go | 9 +++++++++ xdm/parse.go | 13 ++++++++++--- xdm/record.go | 9 +++++++-- xdm/retain_test.go | 30 ++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/internal/xmltok/xmltok.go b/internal/xmltok/xmltok.go index 1082e91c..dd1ec78c 100644 --- a/internal/xmltok/xmltok.go +++ b/internal/xmltok/xmltok.go @@ -224,6 +224,15 @@ func NewDecoderSize(r io.Reader, size int) *Decoder { // minWindow is the smallest read window NewDecoderSize uses. const minWindow = 512 +// TakeText hands the caller the CharData last returned, to keep: the Decoder +// builds its next token in a new buffer rather than over it. It is called +// only directly after RawToken returned a CharData. +func (d *Decoder) TakeText() []byte { + b := d.tokText + d.scratch = nil + return b +} + // IsVersion11 reports whether the document declared version="1.1". The // version comes from the XML declaration and nowhere else, so there is no // setter: a caller able to assert a version could give a 1.0 document 1.1's diff --git a/xdm/parse.go b/xdm/parse.go index 57bbb994..242cf396 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -472,7 +472,7 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { sawPrologToken = true continue } - run.add(cur, t) + run.add(cur, t, dec) case *xml.Comment: t := *tok @@ -876,7 +876,7 @@ type textRun struct { stored bool // the run did not start with white space and is in the store } -func (r *textRun) add(parent *Node, b []byte) { +func (r *textRun) add(parent *Node, b []byte, dec *xml.Decoder) { if len(b) == 0 { return } @@ -890,7 +890,14 @@ func (r *textRun) add(parent *Node, b []byte) { // A run that starts with anything but white space can be neither // stripped nor shared, so it goes straight to the store. if !onlySpace(b) { - r.node.v0, r.node.v1 = r.node.tree.text.addBytes(b, 0) + if len(b) >= textOwnFrom && cap(b)-len(b) <= len(b)/8 { + // Long enough for a block of its own, and not much larger + // than its value: the decoder's buffer becomes the block, + // rather than being copied into one. + r.node.v0, r.node.v1 = r.node.tree.text.keep(dec.TakeText()) + } else { + r.node.v0, r.node.v1 = r.node.tree.text.addBytes(b, 0) + } r.stored = true return } diff --git a/xdm/record.go b/xdm/record.go index f2a59ddf..4da1ac81 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1534,12 +1534,17 @@ func (s *textStore) add(v string) (uint32, uint32) { // Every unsafe.String in the module is over append-only bytes (the // tokeniser's arena, this store), which keeps that true. if len(v) >= textOwnFrom { - s.newBlock(unsafe.Slice(unsafe.StringData(v), len(v))) - return uint32(len(s.blocks)-1) << textShift, uint32(len(v)) + return s.keep(unsafe.Slice(unsafe.StringData(v), len(v))) } return s.addBytes(unsafe.Slice(unsafe.StringData(v), len(v)), 0) } +// keep stores b, which nothing else writes to, as a block of its own. +func (s *textStore) keep(b []byte) (uint32, uint32) { + s.newBlock(b) + return uint32(len(s.blocks)-1) << textShift, uint32(len(b)) +} + // addBytes stores b with room for extra bytes after it, which extend uses // while nothing else has been stored behind it. func (s *textStore) addBytes(b []byte, extra int) (uint32, uint32) { diff --git a/xdm/retain_test.go b/xdm/retain_test.go index 079e89b2..5d4b6434 100644 --- a/xdm/retain_test.go +++ b/xdm/retain_test.go @@ -52,3 +52,33 @@ func TestNamespaceURIDoesNotPinArena(t *testing.T) { t.Errorf("a namespace declaration costs %d bytes per kept tree, want under 2 KB", ns-plain) } } + +// A text value long enough for a block of its own was built in the +// tokeniser's buffer and then copied into the block: two allocations of its +// size. The buffer is handed over now, when it is not much larger than the +// value. Measured on the e-invoice corpus, whose attachments are base64 text: +// 4.10 MB allocated a pass before, 3.49 MB after. +func TestLongTextIsNotCopied(t *testing.T) { + if raceEnabled { + t.Skip("allocation sizes are not stable under -race") + } + long := strings.Repeat("QUJDRA==", 25000) // 200 KB + doc := `short` + long + `after` + var tree *Tree + per := allocated(func() { + var err error + if tree, err = Parse(strings.NewReader(doc), ParseOptions{}); err != nil { + t.Fatal(err) + } + }) + t.Logf("%d bytes allocated for a %d-byte text", per, len(long)) + // The buffer's growth to the value's size is most of the rest: 323 KB + // measured, against 528 KB with the copy. + if per > uint64(len(long))*2 { + t.Errorf("parsing a %d-byte text allocated %d bytes, want under 2x: one buffer, not a buffer and a copy", len(long), per) + } + e := kids(kids(tree.Root)[0]) + if kids(e[0])[0].Value() != "short" || kids(e[1])[0].Value() != long || kids(e[2])[0].Value() != "after" { + t.Errorf("values read back wrong") + } +} From ef5f06400c4552988fbdee0208368377eea9c482 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:21:53 -0400 Subject: [PATCH 22/98] xpath: key distinct-values' promoted doubles only once one is needed Every exact value kept also wrote the double and float keys it promotes to, though only a float or double looks them up. They are now written when the first one arrives. distinct-values(1 to 100000) benchmark: 76.8 -> 53.2 MB/op. --- xpath/fn_misc_test.go | 30 ++++++++++++++++++++++++++++++ xpath/fn_seq.go | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/xpath/fn_misc_test.go b/xpath/fn_misc_test.go index 518208bc..8cc5cf9a 100644 --- a/xpath/fn_misc_test.go +++ b/xpath/fn_misc_test.go @@ -867,6 +867,10 @@ func TestDistinctValuesNumericPromotion(t *testing.T) { {`count(distinct-values((1.2, xs:double(1.2))))`, "1"}, {`count(distinct-values((1, xs:double(1))))`, "1"}, {`count(distinct-values((1, 1.0, xs:double(1), xs:float(1))))`, "1"}, + // The promoted keys are recorded from the first double on, for the + // exact values kept before it and after it alike. + {`count(distinct-values((1.2, xs:double(5), 1.3, xs:double(1.2), xs:double(1.3))))`, "3"}, + {`count(distinct-values((xs:double(1.2), 1.2)))`, "1"}, // Genuinely different values are still distinguished. {`count(distinct-values((1, 2, 3)))`, "3"}, {`count(distinct-values((1.2, 1.3)))`, "2"}, @@ -1012,6 +1016,32 @@ func TestDistinctValuesLargeHomogeneous(t *testing.T) { if got := evalStrXSLT(t, testDoc, `count(distinct-values((1 to 1000, xs:float(1.0))))`); got != "1000" { t.Errorf("range plus equal float = %q, want 1000", got) } + + // With no float or double among them, exact values are keyed once, not + // also under the double and float they promote to. 2000 integers + // measured 14,061 allocations with the promoted keys and 6,054 without. + ctx := NewContext(nil, Builtins()) + v, err := Eval(`1000 to 2999`, ctx, nil) + if err != nil { + t.Fatal(err) + } + ctx.Vars["v"] = v + dv := MustCompile(`distinct-values($v)`, nil) + var n int + allocs := testing.AllocsPerRun(5, func() { + got, err := dv.Eval(ctx) + if err != nil { + t.Fatal(err) + } + n = len(got) + }) + if n != 2000 { + t.Fatalf("distinct-values($v): %d items, want 2000", n) + } + t.Logf("%.0f allocations", allocs) + if allocs > 10000 { + t.Errorf("distinct-values($v) allocated %.0f times, want at most 10,000", allocs) + } } // benchEvalXSLT is evalStrXSLT for a benchmark, which cannot use the *testing.T diff --git a/xpath/fn_seq.go b/xpath/fn_seq.go index 6763672a..d014e224 100644 --- a/xpath/fn_seq.go +++ b/xpath/fn_seq.go @@ -352,6 +352,21 @@ func fnDistinctValues(ctx *Context, args []xdm.Sequence) (xdm.Sequence, error) { // the pairwise scan walks neither the non-numeric results nor the exact // numerics, which the hash key already settles. var inexact []*xdm.Atomic + // A later double is compared against an exact value by promoting it to + // double, so each exact value kept records which double (and float) it + // promotes to as well. That is the key the double will look itself up + // under, and it is what makes xs:decimal("1.2") and xs:double(1.2) one + // value whichever order they arrive in. Only a float or a double looks + // those keys up, so they are recorded once the first one arrives, and + // the exact values kept until then wait in pending. + var pending []*big.Rat + promoting := false + promote := func(r *big.Rat) { + f, _ := r.Float64() + seen["xd\x00"+strconv.FormatFloat(f, 'g', 17, 64)] = true + seen["xf\x00"+strconv.FormatFloat( + float64(float32(f)), 'g', 17, 32)] = true + } for _, it := range atoms { a := it.(*xdm.Atomic) @@ -397,15 +412,11 @@ func fnDistinctValues(ctx *Context, args []xdm.Sequence) (xdm.Sequence, error) { continue } seen[key] = true - // A later double is compared against this value by promoting - // it to double, so record which double it promotes to as - // well. That is the key the double will look itself up under, - // and it is what makes xs:decimal("1.2") and xs:double(1.2) - // one value whichever order they arrive in. - f, _ := r.Float64() - seen["xd\x00"+strconv.FormatFloat(f, 'g', 17, 64)] = true - seen["xf\x00"+strconv.FormatFloat( - float64(float32(f)), 'g', 17, 32)] = true + if promoting { + promote(r) + } else { + pending = append(pending, r) + } out = append(out, a) continue } @@ -414,7 +425,14 @@ func fnDistinctValues(ctx *Context, args []xdm.Sequence) (xdm.Sequence, error) { // the scan. An exact value e is equal to this value under eq // exactly when e promoted to this type equals it -- so the double // (or float) that e promotes to is the key to look up, and each - // exact value recorded both when it was kept. + // exact value records both from here on. + if !promoting { + promoting = true + for _, r := range pending { + promote(r) + } + pending = nil + } var key string if a.Type == xdm.TypeFloat { key = "xf\x00" + strconv.FormatFloat( From 14c84b525f64010ff3ab353d3b970d7695bf5848 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:22:29 -0400 Subject: [PATCH 23/98] xsd: parse bound facets once with the memoised chain checkBounds parsed every min/max bound into a big.Rat for every value. The chain memo now carries the parsed bounds. BenchmarkValidateInstance: 183,403 -> 163,398 B/op, 5,036 -> 4,036 allocs/op. --- xsd/bound_rat_alloc_test.go | 28 ++++++++++++++++++++++++++++ xsd/component.go | 2 +- xsd/facet.go | 7 +++++++ xsd/validate_simple.go | 37 +++++++++++++++++++++++++++++++------ 4 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 xsd/bound_rat_alloc_test.go diff --git a/xsd/bound_rat_alloc_test.go b/xsd/bound_rat_alloc_test.go new file mode 100644 index 00000000..e4b40fcf --- /dev/null +++ b/xsd/bound_rat_alloc_test.go @@ -0,0 +1,28 @@ +package xsd + +import ( + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// The bound facets are parsed into rationals once, with the memoised chain. +// checkBounds parsed every bound again for every value validated: 18% of +// BenchmarkValidateInstance, whose one bound is minInclusive="0". +func TestBoundFacetsParsedOnce(t *testing.T) { + s := mustParseSchema(t, ` + + + + `) + typ := xdm.QName{Local: "T"} + for v, ok := range map[string]bool{"12.5": true, "-1000.5": true, "1000.5": false, "-1000.6": false} { + if err := s.ValidateValue(v, typ); (err == nil) != ok { + t.Errorf("ValidateValue(%q) = %v, want ok=%v", v, err, ok) + } + } + // 25 allocations per value with the bounds re-parsed, 11 without. + if n := testing.AllocsPerRun(20, func() { _ = s.ValidateValue("12.5", typ) }); n > 15 { + t.Errorf("ValidateValue made %v allocations, want at most 15; are the bounds re-parsed?", n) + } +} diff --git a/xsd/component.go b/xsd/component.go index 5f59750e..a6bc29b6 100644 --- a/xsd/component.go +++ b/xsd/component.go @@ -645,7 +645,7 @@ func computeChainFacts(t *SimpleType) *chainFacts { for cur := t; cur != nil && !seen[cur]; { seen[cur] = true if cur.Facets != nil { - f.steps = append(f.steps, facetStep{typ: cur, facets: cur.Facets}) + f.steps = append(f.steps, facetStep{typ: cur, facets: cur.Facets, rats: boundRats(cur.Facets)}) } if cur.Name.URI == NSSchema { if cur.Name.Local == "integer" { diff --git a/xsd/facet.go b/xsd/facet.go index 1ec4f0e8..bde41380 100644 --- a/xsd/facet.go +++ b/xsd/facet.go @@ -2,6 +2,7 @@ package xsd import ( "fmt" + "math/big" "strings" "unicode/utf8" @@ -500,6 +501,12 @@ func (f *FacetSet) whiteSpaceOnly() bool { type facetStep struct { typ *SimpleType facets *FacetSet + // rats are the bound facets parsed as rationals, in the order + // MinInclusive, MaxInclusive, MinExclusive, MaxExclusive, with nil for + // a bound that is absent or not a rational. They are parsed once with + // the memoised chain; checkBounds parsed them again for every value. + // The whole array is nil when the step has no rational bound. + rats *[4]*big.Rat } // facetChain returns the derivation steps from t up to its primitive, nearest diff --git a/xsd/validate_simple.go b/xsd/validate_simple.go index 6d578d04..d276da34 100644 --- a/xsd/validate_simple.go +++ b/xsd/validate_simple.go @@ -829,7 +829,12 @@ func checkBounds(steps []facetStep, normalized, prim string) error { for _, st := range steps { f := st.facets - // cmp orders the instance value against a bound literal. + var rats [4]*big.Rat + if st.rats != nil { + rats = *st.rats + } + // cmp orders the instance value against a bound literal, using + // the bound pre-parsed with the chain when there is one. // // The floating types have three values with no rational form. // §3.3.5/§3.3.6 order them -INF < every finite value < INF and @@ -838,7 +843,7 @@ func checkBounds(steps []facetStep, normalized, prim string) error { // an unparsable value as "no opinion" is what let INF satisfy // maxInclusive="100": being outside every bound the schema can // write is exactly what makes it invalid. - cmp := func(lex *string) (int, bool) { + cmp := func(lex *string, pre *big.Rat) (int, bool) { if lex == nil { return 0, false } @@ -878,6 +883,9 @@ func checkBounds(steps []facetStep, normalized, prim string) error { if !finite { return 0, false } + if pre != nil { + return val.Cmp(pre), true + } b, ok := new(big.Rat).SetString(lexs) if !ok { return 0, false @@ -904,19 +912,19 @@ func checkBounds(steps []facetStep, normalized, prim string) error { } continue } - if c, ok := cmp(f.MinInclusive); ok && c < 0 { + if c, ok := cmp(f.MinInclusive, rats[0]); ok && c < 0 { return facetError(st.typ, FacetMinInclusive, "%s is less than %s", normalized, *f.MinInclusive) } - if c, ok := cmp(f.MaxInclusive); ok && c > 0 { + if c, ok := cmp(f.MaxInclusive, rats[1]); ok && c > 0 { return facetError(st.typ, FacetMaxInclusive, "%s is greater than %s", normalized, *f.MaxInclusive) } - if c, ok := cmp(f.MinExclusive); ok && c <= 0 { + if c, ok := cmp(f.MinExclusive, rats[2]); ok && c <= 0 { return facetError(st.typ, FacetMinExclusive, "%s is not greater than %s", normalized, *f.MinExclusive) } - if c, ok := cmp(f.MaxExclusive); ok && c >= 0 { + if c, ok := cmp(f.MaxExclusive, rats[3]); ok && c >= 0 { return facetError(st.typ, FacetMaxExclusive, "%s is not less than %s", normalized, *f.MaxExclusive) } @@ -924,6 +932,23 @@ func checkBounds(steps []facetStep, normalized, prim string) error { return nil } +// boundRats parses a facet set's bounds for facetStep.rats. +func boundRats(f *FacetSet) *[4]*big.Rat { + var out *[4]*big.Rat + for i, lex := range [4]*string{f.MinInclusive, f.MaxInclusive, f.MinExclusive, f.MaxExclusive} { + if lex == nil { + continue + } + if r, ok := new(big.Rat).SetString(trimXMLSpace(*lex)); ok { + if out == nil { + out = new([4]*big.Rat) + } + out[i] = r + } + } + return out +} + // hasBoundFacet reports whether any step of the chain carries one of the four // bound facets, which is all checkBounds and its temporal and duration // variants consult. From a5ab73679efa8d7ca35289e30a4c75f5dabce118 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:22:31 -0400 Subject: [PATCH 24/98] xdm: trim a parsed tree's last text block Text blocks double, so a parsed tree's last one was on average a quarter empty. It is copied to its length at freeze. DocBook corpus: 12.29 -> 12.03 MB retained. --- xdm/parse.go | 1 + xdm/record.go | 12 ++++++++++++ xdm/retain_test.go | 16 ++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/xdm/parse.go b/xdm/parse.go index 242cf396..1ff3c3ce 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -713,6 +713,7 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { } tree.closeAll() tree.frozen = true + tree.text.trim() if trackPos { // The decoder stops reading at the end of the root element, so the diff --git a/xdm/record.go b/xdm/record.go index 4da1ac81..f4f0b982 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1566,6 +1566,18 @@ func (s *textStore) addBytes(b []byte, extra int) (uint32, uint32) { return uint32(k)< len(b) { + s.blocks[s.cur-1] = append([]byte(nil), b...) + } +} + func (s *textStore) newBlock(b []byte) { if len(s.blocks) >= 1<<(32-textShift) { panic("xdm: a tree's text exceeds the text store's addressable blocks") diff --git a/xdm/retain_test.go b/xdm/retain_test.go index 5d4b6434..29c4de79 100644 --- a/xdm/retain_test.go +++ b/xdm/retain_test.go @@ -82,3 +82,19 @@ func TestLongTextIsNotCopied(t *testing.T) { t.Errorf("values read back wrong") } } + +// A parsed tree's last text block was kept at the size it doubled to, on +// average a quarter empty. Measured on the DocBook corpus: 12.29 MB retained +// for 591 trees before the trim, 12.03 MB after. +func TestParsedTextBlockIsTrimmed(t *testing.T) { + tree, err := ParseString(``+strings.Repeat(`

some text

`, 200)+``, ParseOptions{}) + if err != nil { + t.Fatal(err) + } + s := &tree.text + b := s.blocks[s.cur-1] + // Rounding to the allocator's size class may leave a little. + if slack := cap(b) - len(b); slack > len(b)/8 { + t.Errorf("last text block holds %d bytes in %d", len(b), cap(b)) + } +} From 4440665fabb85d5416e84acecf892df860631f56 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:26:00 -0400 Subject: [PATCH 25/98] xdm: drop the build-only indexes when a parse freezes its tree The name and namespace-frame indexes serve only building, but a parsed tree kept them. They are dropped at freeze and rebuilt by a later rename or clone. DocBook corpus: 12.03 -> 11.07 MB retained. --- xdm/clone_test.go | 4 ++-- xdm/parse.go | 5 +++++ xdm/record.go | 25 ++++++++++++++++++------- xdm/retain_test.go | 26 ++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/xdm/clone_test.go b/xdm/clone_test.go index 6a967d00..5f3abbbd 100644 --- a/xdm/clone_test.go +++ b/xdm/clone_test.go @@ -105,8 +105,8 @@ func TestCloneInternsPastSmallNames(t *testing.T) { } got := xdmclone.Clone(tree.Root, xdmclone.Options{})(tree.Root).(*Node) c := got.Tree() - if c == nil || c.nameIx == nil { - t.Fatalf("clone has no name index to exercise (tree %v)", c) + if c == nil || len(c.names) <= smallNames { + t.Fatalf("clone has too few names to need an index (tree %v)", c) } old := c.intern(QName{Local: "e3"}) if c.names[old] != (QName{Local: "e3"}) { diff --git a/xdm/parse.go b/xdm/parse.go index 1ff3c3ce..36b99609 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -714,6 +714,11 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { tree.closeAll() tree.frozen = true tree.text.trim() + // The indexes only building uses go; a tree renamed or cloned later + // rebuilds the ones it needs. + tree.nameIx, tree.nameHot = nil, nil + tree.frameOne, tree.frameMany = nil, nil + tree.open = nil if trackPos { // The decoder stops reading at the end of the root element, so the diff --git a/xdm/record.go b/xdm/record.go index f4f0b982..fe58945a 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -358,6 +358,11 @@ func (t *Tree) intern(q QName) uint32 { if q == (QName{}) { return 0 } + // A parsed tree drops its index when it is frozen, and a clone of one + // copies none: it is rebuilt here, on first use. + if t.nameIx == nil && len(t.names) > smallNames { + t.indexNames() + } // A small tree -- a constructed fragment, mostly -- has a handful of // names, and finding one by scanning costs less than building a map. if t.nameIx == nil { @@ -372,11 +377,7 @@ func (t *Tree) intern(q QName) uint32 { i := uint32(len(t.names)) t.names = append(t.names, q) if len(t.names) > smallNames { - t.nameIx = make(map[QName]uint32, 2*len(t.names)) - t.nameHot = new([64]uint32) - for j, x := range t.names[1:] { - t.nameIx[x] = uint32(j + 1) - } + t.indexNames() } return i } @@ -399,6 +400,15 @@ func (t *Tree) intern(q QName) uint32 { return i } +// indexNames builds the name index over the names the tree holds. +func (t *Tree) indexNames() { + t.nameIx = make(map[QName]uint32, 2*len(t.names)) + t.nameHot = new([64]uint32) + for j, x := range t.names[1:] { + t.nameIx[x] = uint32(j + 1) + } +} + // nameHash is a cheap hash of q for nameHot: the lengths and the ends of the // local name, which tell most names in a document apart. func nameHash(q QName) int { @@ -1075,8 +1085,9 @@ func (t *Tree) internFrame(ns []nsBinding) uint32 { } return t.addFrame(ns) } - if len(t.frames) == smallFrames+1 && t.frameOne == nil && t.frameMany == nil { - // Past the scan's reach: index what is there. + if len(t.frames) > smallFrames && t.frameOne == nil && t.frameMany == nil { + // Past the scan's reach: index what is there. A parsed tree drops + // the indexes when it is frozen, so this also rebuilds them. for i := 1; i < len(t.frames); i++ { t.indexFrame(uint32(i)) } diff --git a/xdm/retain_test.go b/xdm/retain_test.go index 29c4de79..3f0e5b5f 100644 --- a/xdm/retain_test.go +++ b/xdm/retain_test.go @@ -1,6 +1,7 @@ package xdm import ( + "fmt" "runtime" "strings" "testing" @@ -98,3 +99,28 @@ func TestParsedTextBlockIsTrimmed(t *testing.T) { t.Errorf("last text block holds %d bytes in %d", len(b), cap(b)) } } + +// The name and frame indexes serve only building; a parsed tree kept them +// for its life. Measured on the DocBook corpus: 12.03 MB retained for 591 +// trees with them, 11.07 MB without. A tree changed after the parse rebuilds +// what it needs. +func TestFrozenTreeDropsBuildIndexes(t *testing.T) { + var b strings.Builder + b.WriteString("") + for i := range 2 * smallNames { + fmt.Fprintf(&b, ``, i, i, i) + } + b.WriteString("") + tree, err := ParseString(b.String(), ParseOptions{}) + if err != nil { + t.Fatal(err) + } + if tree.nameIx != nil || tree.nameHot != nil || tree.frameOne != nil || tree.frameMany != nil || tree.open != nil { + t.Errorf("a parsed tree kept its build indexes") + } + e := kids(kids(tree.Root)[0]) + e[0].SetName(QName{Local: "e5"}) + if e[0].name != e[5].name || tree.nameIx == nil { + t.Errorf("renaming to an existing name gave index %d, want %d (index rebuilt: %v)", e[0].name, e[5].name, tree.nameIx != nil) + } +} From e3b628880250ae91c3c0d2582c4059e7abf4daec Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:26:27 -0400 Subject: [PATCH 26/98] xslt: pass with-param values as a slice, not two maps evalParams built a normal and a tunnel map per call, the tunnel one even when empty: 3 allocations for two parameters. A slice of pairs is 1; DocBook section.001 transform 20.1 -> 19.0 MB allocated. --- xslt/apply.go | 77 +++++++++++++++++++++++++++++++++-------- xslt/paramalloc_test.go | 47 +++++++++++++++++++++++++ xslt/pattern30.go | 2 +- xslt/runtime.go | 6 ++-- xslt/transform.go | 8 ++--- 5 files changed, 117 insertions(+), 23 deletions(-) create mode 100644 xslt/paramalloc_test.go diff --git a/xslt/apply.go b/xslt/apply.go index 069b1913..79942a3e 100644 --- a/xslt/apply.go +++ b/xslt/apply.go @@ -142,7 +142,7 @@ func (i *applyTemplatesInstr) effectiveMode(rt *runtime) string { // applyToNode selects and runs the best-matching template for one node. func applyToNode(rt *runtime, node *xdm.Node, mode string, - params map[string]xdm.Sequence, tunnels map[string]xdm.Sequence, + params, tunnels paramValues, out *outputBuilder) error { // XTTE3100/XTTE3110 are stated against "an xsl:apply-templates @@ -342,7 +342,7 @@ func (t *Template) matchesMode(mode string) bool { // stylesheet wrote no rule for. Dropping them made those parameters take their // defaults instead. func applyBuiltInRule(rt *runtime, node *xdm.Node, mode string, - params, tunnels map[string]xdm.Sequence, out *outputBuilder) error { + params, tunnels paramValues, out *outputBuilder) error { // Section 6.6: xsl:mode/@on-no-match selects which set of built-in rules // is in force. text-only-copy is the default and the historical XSLT 1.0 @@ -434,14 +434,14 @@ func applyBuiltInRule(rt *runtime, node *xdm.Node, mode string, // runTemplate executes a template with the supplied parameters, in a frame // of its own (see locals.go). func runTemplate(rt *runtime, t *Template, - params map[string]xdm.Sequence, tunnels map[string]xdm.Sequence, + params, tunnels paramValues, out *outputBuilder) error { defer rt.leaveFrame(rt.enterFrame()) return runTemplateBody(rt, t, params, tunnels, out) } func runTemplateBody(rt *runtime, t *Template, - params map[string]xdm.Sequence, tunnels map[string]xdm.Sequence, + params, tunnels paramValues, out *outputBuilder) error { // An abstract template fails before anything else it declares is @@ -505,8 +505,8 @@ func runTemplateBody(rt *runtime, t *Template, for k, v := range sub.tunnel { merged[k] = v } - for k, v := range tunnels { - merged[k] = v + for _, t := range tunnels { + merged[t.name] = t.value } n.tunnel = merged sub = &n @@ -520,7 +520,7 @@ func runTemplateBody(rt *runtime, t *Template, // declaring tunnel="yes" keeps its default when the caller passes an // ordinary parameter of that name. if !p.Tunnel { - if v, ok := params[key]; ok { + if v, ok := params.get(key); ok { v, err := bindParam(p, v, t) if err != nil { return err @@ -615,22 +615,69 @@ func templateLabel(t *Template) string { return "(anonymous)" } +// paramValues is the parameters one invocation supplies, keyed by Clark +// name. A call passes one or two, so a slice searched in order costs one +// small allocation where a map cost a header and a bucket group. +type paramValues []paramValue + +type paramValue struct { + name string + value xdm.Sequence +} + +// get returns the value supplied for name. The last binding wins, as a map +// assignment would. +func (ps paramValues) get(name string) (xdm.Sequence, bool) { + for i := len(ps) - 1; i >= 0; i-- { + if ps[i].name == name { + return ps[i].value, true + } + } + return nil, false +} + +// paramsOf converts a caller's parameter map. +func paramsOf(m map[string]xdm.Sequence) paramValues { + if m == nil { + return nil + } + ps := make(paramValues, 0, len(m)) + for k, v := range m { + ps = append(ps, paramValue{k, v}) + } + return ps +} + // evalParams evaluates xsl:with-param children, separating tunnel parameters. -func evalParams(rt *runtime, params []*Variable) (map[string]xdm.Sequence, map[string]xdm.Sequence, error) { +// With any parameters at all, tunnel is non-nil even when empty, which tells +// xsl:next-match not to inherit the current tunnel set. +func evalParams(rt *runtime, params []*Variable) (paramValues, paramValues, error) { if len(params) == 0 { return nil, nil, nil } - normal := map[string]xdm.Sequence{} - tunnel := map[string]xdm.Sequence{} + nt := 0 + for _, p := range params { + if p.Tunnel { + nt++ + } + } + var normal paramValues + if nt < len(params) { + normal = make(paramValues, 0, len(params)-nt) + } + tunnel := paramValues{} + if nt > 0 { + tunnel = make(paramValues, 0, nt) + } for _, p := range params { v, err := evalVariable(p, rt) if err != nil { return nil, nil, fmt.Errorf("evaluating parameter $%s: %w", p.Name.Lexical(), err) } if p.Tunnel { - tunnel[p.Name.Clark()] = v + tunnel = append(tunnel, paramValue{p.Name.Clark(), v}) } else { - normal[p.Name.Clark()] = v + normal = append(normal, paramValue{p.Name.Clark(), v}) } } return normal, tunnel, nil @@ -1022,7 +1069,7 @@ func (i *nextMatchInstr) Execute(rt *runtime, out *outputBuilder) error { // builtInDescend applies the mode's rules to node's children, optionally // copying text and attribute values through as the text-only rule does. func builtInDescend(rt *runtime, node *xdm.Node, mode string, - params, tunnels map[string]xdm.Sequence, out *outputBuilder, + params, tunnels paramValues, out *outputBuilder, copyText bool) error { switch node.Kind() { @@ -1059,7 +1106,7 @@ func builtInDescend(rt *runtime, node *xdm.Node, mode string, // Every other kind of node produces the empty sequence, which is what falling // off the switch does. func builtInSkipDescend(rt *runtime, node *xdm.Node, mode string, - params, tunnels map[string]xdm.Sequence, out *outputBuilder) error { + params, tunnels paramValues, out *outputBuilder) error { if node.Kind() != xdm.KindDocument && node.Kind() != xdm.KindElement { return nil @@ -1098,7 +1145,7 @@ func builtInSkipDescend(rt *runtime, node *xdm.Node, mode string, // Attributes do not: the rule processes them like any other child in the // mode, and the built-in attribute rule is what puts their values back. func builtInShallowCopy(rt *runtime, node *xdm.Node, mode string, - params, tunnels map[string]xdm.Sequence, out *outputBuilder) error { + params, tunnels paramValues, out *outputBuilder) error { switch node.Kind() { case xdm.KindDocument: diff --git a/xslt/paramalloc_test.go b/xslt/paramalloc_test.go new file mode 100644 index 00000000..205ac385 --- /dev/null +++ b/xslt/paramalloc_test.go @@ -0,0 +1,47 @@ +package xslt + +import ( + "context" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// paramCallAllocs is the allocations per xsl:call-template of a stylesheet +// that makes 200 calls passing WITH, less those of one making the calls with +// no parameters at all. +func paramCallAllocs(t *testing.T, with string) float64 { + t.Helper() + run := func(with string) float64 { + src := strings.ReplaceAll(` +WITH + +`, "WITH", with) + d, err := xdm.ParseString(src, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + st, err := Compile(d.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + return testing.AllocsPerRun(5, func() { + if _, err := st.Transform(context.Background(), nil, TransformOptions{InitialTemplate: "main"}); err != nil { + t.Fatal(err) + } + }) + } + return (run(with) - run("")) / 200 +} + +// Parameters an invocation supplies are a small slice, not two maps: the two +// cost a header and a bucket group each per call, the tunnel one even when no +// tunnel parameter was passed. +func TestWithParamAllocs(t *testing.T) { + two := `` + // One for the slice; the two maps cost three. + if got := paramCallAllocs(t, two); got > 2 { + t.Errorf("two xsl:with-param cost %.1f allocations per call, want at most 2", got) + } +} diff --git a/xslt/pattern30.go b/xslt/pattern30.go index f3ef3e9c..24e52369 100644 --- a/xslt/pattern30.go +++ b/xslt/pattern30.go @@ -519,7 +519,7 @@ func (p *Pattern) matchesAtomicItem(item xdm.Item, ctx *xpath.Context) (bool, er // phrased about nodes, and none of them has a meaning for a value with no // children, no name and no ancestry. func applyToAtomic(rt *runtime, item xdm.Item, mode string, - params, tunnels map[string]xdm.Sequence, out *outputBuilder) error { + params, tunnels paramValues, out *outputBuilder) error { t, next := rt.sheet.findAtomicTemplateFrom(item, mode, rt.ctx, 0) if t == nil { diff --git a/xslt/runtime.go b/xslt/runtime.go index 9c42ed2c..30773512 100644 --- a/xslt/runtime.go +++ b/xslt/runtime.go @@ -268,8 +268,8 @@ type selection struct { // mode, params and tunnels are carried so a resumed dispatch behaves // like the original one. mode string - params map[string]xdm.Sequence - tunnels map[string]xdm.Sequence + params paramValues + tunnels paramValues // item is the item the rule matched, kept so xsl:next-match can tell // whether it is still looking at it. Section 6.7 makes the current // template rule and the context item two separate conditions, and an @@ -354,7 +354,7 @@ func (rt *runtime) withCurrent(item xdm.Item, pos, size int) *runtime { // withSelection records the template-selection state for the body about to run. func (rt *runtime) withSelection(t *Template, next int, mode string, - params, tunnels map[string]xdm.Sequence) *runtime { + params, tunnels paramValues) *runtime { var item xdm.Item if rt.ctx != nil { item = rt.ctx.Item diff --git a/xslt/transform.go b/xslt/transform.go index 1033e75e..fce40865 100644 --- a/xslt/transform.go +++ b/xslt/transform.go @@ -532,7 +532,7 @@ func (s *Stylesheet) Transform(ctx context.Context, source *xdm.Node, opts Trans } } if err := runTemplate(rt, t, - opts.InitialTemplateParams, opts.InitialTemplateTunnelParams, + paramsOf(opts.InitialTemplateParams), paramsOf(opts.InitialTemplateTunnelParams), out); err != nil { return nil, err } @@ -620,7 +620,7 @@ func (s *Stylesheet) Transform(ctx context.Context, source *xdm.Node, opts Trans "parameter $%s", code, opts.InitialTemplate, p.Name.Lexical()) } if err := runTemplate(rt, t, - opts.InitialTemplateParams, opts.InitialTemplateTunnelParams, + paramsOf(opts.InitialTemplateParams), paramsOf(opts.InitialTemplateTunnelParams), out); err != nil { return nil, err } @@ -699,7 +699,7 @@ func (s *Stylesheet) Transform(ctx context.Context, source *xdm.Node, opts Trans initialMode = want } if err := applyInitialSelection(rt, source, opts.InitialMatchSelection, - initialMode, opts.InitialModeParams, opts.InitialModeTunnelParams, + initialMode, paramsOf(opts.InitialModeParams), paramsOf(opts.InitialModeTunnelParams), out); err != nil { return nil, err } @@ -1485,7 +1485,7 @@ func SerializeAsXML(r *Result) string { // position and size of each item have to be set the way xsl:apply-templates // sets them, since a template rule reached this way may call fn:position(). func applyInitialSelection(rt *runtime, source *xdm.Node, sel xdm.Sequence, - mode string, params, tunnels map[string]xdm.Sequence, + mode string, params, tunnels paramValues, out *outputBuilder) error { if sel == nil { From 599b2f9f692a91c52c2faa7dda852b8ce902378b Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 11:27:50 -0400 Subject: [PATCH 27/98] serialize: stream the json and adaptive methods to the writer Both methods built the whole document as a string before writing it, 7.5-12.6x the output size, and a node inside JSON was serialised to a string and then escaped into a second one. xpath gains SerializeJSONTo and SerializeAdaptiveTo, which write through the serializer's writer; a JSON node is escaped as it is written. Adaptive output under normalization-form keeps the string, because norm's streaming Writer panics on long combining runs split across writes. fn:serialize's text and attribute escaping and prefixed names now write to the sink in pieces instead of building a string per node. --- xpath/fn_serialize.go | 339 ++++++++++++++++++++++------- xpath/fn_serialize_stream_test.go | 95 ++++++++ xslt/serialize.go | 65 ++++-- xslt/serialize_streamalloc_test.go | 31 +++ 4 files changed, 430 insertions(+), 100 deletions(-) create mode 100644 xpath/fn_serialize_stream_test.go diff --git a/xpath/fn_serialize.go b/xpath/fn_serialize.go index fe9f5fae..0dac0e69 100644 --- a/xpath/fn_serialize.go +++ b/xpath/fn_serialize.go @@ -1,7 +1,9 @@ package xpath import ( + "bufio" "fmt" + "io" "math" "sort" "strings" @@ -902,13 +904,13 @@ func serializeNode(sb *serializeSink, n *xdm.Node, opts serializeOptions, depth } case xdm.KindElement: sb.WriteString("<") - sb.WriteString(elementName(n)) + writeName(sb, n) writeNamespaceDecls(sb, n, opts) // Attributes are written in the order the document had them, which is // what a round-trip test compares against. for a := range n.Attrs() { sb.WriteString(" ") - sb.WriteString(elementName(a)) + writeName(sb, a) sb.WriteString(`="`) // A URI-valued attribute under the html or xhtml method has its // non-ASCII characters percent-escaped. XSLT 2.0 @@ -917,9 +919,9 @@ func serializeNode(sb *serializeSink, n *xdm.Node, opts serializeOptions, depth // so the two serialisers disagreed on identical input until now. if opts.escapesURIAttrs() && (opts.method == "html" || opts.method == "xhtml") && isURIAttribute(n.Name().Local, a) { - sb.WriteString(escapeAttr(escapeURIAttribute(a.Value()))) + writeEscaped(sb, escapeURIAttribute(a.Value()), true) } else { - sb.WriteString(escapeAttr(a.Value())) + writeEscaped(sb, a.Value(), true) } sb.WriteString(`"`) } @@ -952,7 +954,7 @@ func serializeNode(sb *serializeSink, n *xdm.Node, opts serializeOptions, depth sb.WriteString(">") } else { sb.WriteString(">") } return @@ -1046,10 +1048,10 @@ func serializeNode(sb *serializeSink, n *xdm.Node, opts serializeOptions, depth writeIndent(sb, depth) } sb.WriteString("") case xdm.KindText: - sb.WriteString(escapeText(opts.normalized(n.Value()))) + writeEscaped(sb, opts.normalized(n.Value()), false) case xdm.KindComment: sb.WriteString(" -The unit-test suite is 2,854 tests. +The unit-test suite is 2,887 tests. The last two rows are not W3C suites but real-world corpora — DocBook xslTNG's diff --git a/docs/migrating-to-v2.md b/docs/migrating-to-v2.md index 29c763c9..363d763b 100644 --- a/docs/migrating-to-v2.md +++ b/docs/migrating-to-v2.md @@ -177,6 +177,7 @@ doc.Finalize() The calls are `AppendElement`, `AppendText`, `AppendComment`, `AppendPI`, `AppendAttr` (before the element's first child), `AddNamespace`, +`AddNamespaces` (several bindings in one call), `AppendCopy` (a deep copy) and `AppendShallowCopy` (the node alone, for a copy you complete yourself). A node is open from the call that made it until you append to something that is not it or inside it; appending to a closed node, @@ -185,7 +186,7 @@ or adding an attribute after a child, panics. `Finalize` closes everything. What may still change after a node is made is its scalar state: `SetName`, `SetValue`, `AppendValue` (merging adjacent text), `SetBaseURI`, `SetDocumentURI`, the typing setters, and the namespace declarations of an -element (`AddNamespace`, `SetNamespaceDecl`, `RemoveNamespaceDecls`) of a tree +element (`AddNamespace`, `AddNamespaces`, `SetNamespaceDecl`, `RemoveNamespaceDecls`) of a tree that is not a parsed document. A parsed document is frozen. `xdm.NewNode(kind, name, value)` still makes a parentless node; it belongs to diff --git a/docs/options.md b/docs/options.md index 954f03c6..907d93e7 100644 --- a/docs/options.md +++ b/docs/options.md @@ -180,7 +180,7 @@ tree, err := xdm.ParseString(src, xdm.ParseOptions{ | `BaseURI` | `string` | none | Recorded on the document node; what `fn:base-uri`, `fn:doc` and relative `xsl:include` resolve against. Set it whenever the document came from somewhere addressable. | | `StripSpace` | `func(QName) bool` | keep all | Drops whitespace-only text nodes for the element names it returns true for. A predicate rather than a bool because `xsl:strip-space` is per-element. | | `AllowDOCTYPE` | `bool` | refuse | Permits a `DOCTYPE` declaration. See below. | -| `TrackPositions` | `bool` | off | Records where each element starts, so a validation error can name a line. Retains the source text for the life of the tree — about 10% more memory, no extra parse time. | +| `TrackPositions` | `bool` | off | Records where each element starts, so a validation error can name a line. Keeps each line's start offset and each element's column, not the source text: about 10% more retained memory (parse-10mb 37.2 → 41.5 MB), no extra parse time. | | `MaxDepth` | `int` | `DefaultMaxDepth` = 1000 | Nesting limit. Deep input is the cheapest route to stack exhaustion. | | `MaxBytes` | `int64` | `DefaultMaxBytes` = 64 MB | Source size limit, enforced on the read. | | `MaxNodes` | `int` | `DefaultMaxNodes` = 10,000,000 | Node count limit, counting attributes and namespaces. | diff --git a/docs/profiling.md b/docs/profiling.md index 294d61d3..181d57e5 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -131,6 +131,9 @@ The profile is flat. The largest single functions are `QName` equality | `Tree.alloc` (result-tree records) | — | 6.2% | | `evalParams` (cumulative) | — | 6.4% | +This profile predates the memory audit: `evalParams` now passes a slice of +pairs, not two maps, and section.001 allocates 15% less (CHANGELOG, *Unreleased*). + No source exceeds a tenth of the objects. DocBook's cost is many small allocations spread over the whole evaluator, and the collector work they cause. @@ -201,7 +204,8 @@ invoice are 20 k and 1.95 MB. Allocations: 42% of the bytes are node-record chunks (`Tree.alloc`), 22% the harness's input copy and 16% the text store. By count, FLWOR tuple binding -(`tuple.bind`) is 27%, but these are small objects and about 2% of CPU. XMark +(`tuple.bind`) was 27% before the memory audit, which shares one names +slice per tuple stream, but these are small objects and about 2% of CPU. XMark at factor 0.1 is a parse benchmark. ### XSD validation and typed validation @@ -248,9 +252,10 @@ objects. By bytes, `Tree.alloc` makes 21%, the clone 17% and the typing 48% of the bytes are the tokenizer's 32 KB string arena, one per document. `newAfterPat` makes 37% of the objects and `strings.Fields` (token normalisation) 12%. The documents are small, so that fixed arena dominates -the bytes and drives the GC share. Sizing the arena to the input was measured -and rejected because of how it moved GC pacing on DocBook (see -[rejected](#measured-and-rejected)). +the bytes and drives the GC share. Sizing the arena to the input was first +rejected because of how it moved GC pacing on DocBook (see +[rejected](#measured-and-rejected)); the memory audit adopted it once +namespace URIs stopped pinning the arena (CHANGELOG, *Unreleased*). ### Parse and Canonical XML @@ -379,7 +384,10 @@ any dropped passing count. the same `GOGC`. A change that cuts retained memory can cost CPU elsewhere: starting the string arena small cut small documents' retained heap 20–37% and cost DocBook 5% CPU (85 GC cycles against 77). Count GC cycles a pass - in both arms. + in both arms. The memory audit raised DocBook xslTNG (593 inputs, one + process) from about 142 to 174 cycles, the same with or without the sized + arena, with peak RSS 1.24–1.35 GB → 1.07–1.15 GB and user CPU within noise + over five alternating rounds. - **The memory profiler costs CPU.** A binary that links `runtime/pprof` samples allocations unless `MemProfileRate` is 0. The CLI does not link it, so a harness that does measures a slightly different program. @@ -477,7 +485,7 @@ longer holds. | Smaller first record chunk (2 records, V15) | Most fragments hold one node, but bytes moved −0.4% to +0.2% and allocations up to +0.6%: the next chunk comes sooner for every other tree | | Chunked result nodes | −0.9% allocations at most, +0.4–0.7% bytes | | String arena starting small | Retained heap −20–37% on small documents, but DocBook +5% CPU: the smaller live heap ran the collector more often (85 cycles against 77). Reverted in `629cf48` | -| String arena sized to the input (V18) | 40 small RELAX NG documents: parse bytes −52%, CPU −20%, retained heap −72%. But DocBook ran about 7% more GC cycles (29.3 → 31.3 a pass), as above. Only the read windows were sized (`9ddd2bc6`) | +| String arena sized to the input (V18) | 40 small RELAX NG documents: parse bytes −52%, CPU −20%, retained heap −72%. But DocBook ran about 7% more GC cycles (29.3 → 31.3 a pass), as above. Only the read windows were sized (`9ddd2bc6`). **Adopted later** in `9e13f70`: with namespace URIs copied out (`18d374d`) the arena no longer pinned trees, and an A/B with and without it gave DocBook the same GC count | | Attribute values passed to xdm without the arena copy (V40) | Parse bytes −2.3 to −2.6%, but `regex-syntax` +2.6% bytes, XRechnung +26 to +53 allocations a pass and more GC cycles (7.2 → 7.6); no CPU gain. Reviving it needs the copy at `textStore.add`, since the values sit in a reused buffer | | End tags checked against a stack of open names (V35) | After the tokenizer's name cache (V34), parse +0.8 to +1.7% CPU and 7 more allocations a parse | | SWAR scan in the tokenizer's `text()` | No gain (round 3, and again on v2): it is already a tight table loop | diff --git a/docs/security.md b/docs/security.md index 34dbbcf6..3bfbb205 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1857,6 +1857,12 @@ Each line names the **direction** of the defect, because that is what decides who was exposed: a *false accept* let an invalid input through, a *false reject* refused a legal one, and *cost* produced the right answer too slowly. +**Eighteenth pass — a memory audit, 2026-10-31.** The rest of that audit was retention and allocation with no hostile driver; it is in CHANGELOG under *Changed — memory*. + +- **`xsd` error messages kept document strings whole** — cost, driven by document data. `fail` and `attrName` quoted names and values in full and the validator keeps up to 100 errors, so 200 foreign attributes in a 1 MB namespace retained 105 MB, and one 15 MB value under a failing rule set retained 1.57 GB (audit 20). Strings and the formatted message are clipped to 1 KB plus "...", and the pattern and list-item errors quote through `truncate`: 0.13 MB. Pinned by `TestValidationMessagesClipDocumentStrings` in `xsd/message_clip_test.go`. +- **The DTD validator built every element's path, quadratic in depth** — cost, driven by document data. The path string was rebuilt at each element whether or not anything failed: a *valid* 990-deep chain of 4 KB names allocated 2 GB. It is rendered from the ancestors only when a failure is recorded, and an IDREF keeps its element rather than a path. Its `#FIXED`, enum and ID values are clipped to 1 KB too. Pinned by `TestPathBuiltOnlyForFailures` and `TestValidityMessagesClipLongValues` in `dtd/`. +- **The nested `fn:transform` cache kept compiles for collected resolvers** — cost, driven by the host. The key held each resolver by identity, so a service that made one per request kept up to 16 dead compiles a stylesheet: 49 MB on DocBook after 20 resolvers. Resolvers are weak keys and `put` drops entries whose resolver is gone: 0.3 MB. Pinned by `TestTransformCache/dead_resolver_dropped`. + **Seventeenth pass — normalize-space built a list of every token, 2026-10-30.** - **`fn:normalize-space` allocated per whitespace token** — cost, driven by document data. It was `strings.Join(splitXMLSpace(s), " ")`, so a value of n tokens built an n-element slice and then the result; a rule set evaluating it 180 times on one 15 MB value allocated 133.5 GB in 20.6 s, and `MaxBytes` charges only the result. It and `fn:tokenize#1` now use `collapseXMLSpace`, one pass into a builder, which returns an already-collapsed value as it is: 0 GB in 2.6 s, and 2.5 GB in 4.2 s for a value that needs collapsing. Pinned by `TestCollapseXMLSpaceMatchesSplitJoin` and `TestCollapseXMLSpaceAllocation` in `xpath/xml_space_test.go`. Fixed in v2.0.3. diff --git a/docs/stats.md b/docs/stats.md index b9784dbe..cfe2db31 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -23,7 +23,7 @@ claim is only as good as the command behind it. | figure | count | counted by | |---|---:|---| -| Unit tests | 2,854 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | +| Unit tests | 2,887 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Fuzz targets | 12 | `grep -rn '^func Fuzz' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Limit boundary tests | 14 | `grep -hc '^func Test' ./*/limits_boundary*_test.go \| awk '{n += $1} END {print n + 0}'` | diff --git a/docs/testing.md b/docs/testing.md index 2eea7e20..88c1c0c4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,7 +21,7 @@ let something through, and the column that matters is the last one. | layer | count | catches | misses | |---|---:|---|---| -| **Unit tests** | 2,854 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** | 2,887 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | | **Limit boundary tests** | 14 tests | an off-by-one or an overflow at the edge of a configurable limit | a limit nobody added to the inventory | | **Race detector** | same tests | shared state a single-goroutine run never reveals | a data race on a path no test walks | | **W3C conformance suites** | 152,241 cases | systematic divergence from the specification | what the suites do not ask about — see below | diff --git a/docs/todo.md b/docs/todo.md index 8c7001db..9b41a69c 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -23,7 +23,7 @@ Current position: | XSLT 3.0 | 99.80% — 11,495 of 11,518 in scope (23 failing); 3 of those need more of the §19.8 streamability analysis | | RELAX NG | 100.00% — 965 of 965 | | Schemas wrongly refused | 7 — 6 on XSD 1.0, 1 on 1.1 | -| Tests | 2,854 `func Test` declarations, clean under `-race` | +| Tests | 2,887 `func Test` declarations, clean under `-race` | Every one of those failures, and why it is still open, is catalogued in [known-gaps.md](known-gaps.md). This file is the forward-looking half — what @@ -150,7 +150,8 @@ silently replaced the character — which is the corruption the MUST exists to prevent, not merely a conformance divergence. `xslt/serialize.go` had the rule right all along, so this was again one rule with two implementations. -Both functions now delegate to `escapeXMLRunes`, which walks runes and decides +Both functions now delegate to `escapeXMLRunes` (since the memory audit, content +escaping streams through `writeEscaped`, its writer form), which walks runes and decides by range rather than by a fixed list: adding U+0085 and U+2028 as two more replacer entries would have closed the named half and left the ranges open. `#x0` is deliberately not escaped — it is not a valid XML character at any diff --git a/docs/xsd.md b/docs/xsd.md index 9f5e85f3..93aecfb2 100644 --- a/docs/xsd.md +++ b/docs/xsd.md @@ -42,7 +42,9 @@ if errors.As(err, &invalid) { The code — `cvc-complex-type.2.4.a`, `cvc-datatype-valid.1.2.1` — is what lets a caller tell the kinds of failure apart without matching on message -text. Message wording is not part of the contract; codes are. +text. Message wording is not part of the contract; codes are. A name or value +the document supplies is clipped to 1 KB plus "..." in the message, so a +hostile input cannot make the retained errors large. There are four ways in, differing only in where the schema comes from: From 4e8f0a532990c188a484809c7d71f01186e92f78 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:11:22 -0400 Subject: [PATCH 40/98] xslt: refuse a result tree that overran MaxNodes instead of returning it truncated --- xpath/context.go | 39 ++++++++++++++++++++++++++++----------- xslt/nodebudget_test.go | 25 +++++++++++++++++++++++++ xslt/transform.go | 8 ++++++++ 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/xpath/context.go b/xpath/context.go index 38a813d1..4ccd67e0 100644 --- a/xpath/context.go +++ b/xpath/context.go @@ -1499,21 +1499,38 @@ func (c *Context) countNodes(n int) error { return nil } if atomic.AddInt64(c.ev().nodes, int64(n)) > MaxNodes { - // XPDY0130 again, and the wording says nodes so that the three - // refusals are not confused with one another. XPath 3.1 §2.3.1 - // sanctions exactly this: "limitations may exist on the maximum - // numbers or sizes of various objects... An error must be raised if - // such a limitation is exceeded [err:XPDY0130]". The sentinel is - // added alongside so a caller can tell a refusal from a fault. See - // xdm.ErrResourceLimit. - return fmt.Errorf( - "XPDY0130: evaluation constructed more than %d result-tree "+ - "nodes; the transformation is building a tree too large to "+ - "hold: %w", MaxNodes, xdm.ErrResourceLimit) + return nodesRefusal() } return nil } +// nodesRefusal is the XPDY0130 the node budget raises. +func nodesRefusal() error { + // XPDY0130 again, and the wording says nodes so that the three + // refusals are not confused with one another. XPath 3.1 §2.3.1 + // sanctions exactly this: "limitations may exist on the maximum + // numbers or sizes of various objects... An error must be raised if + // such a limitation is exceeded [err:XPDY0130]". The sentinel is + // added alongside so a caller can tell a refusal from a fault. See + // xdm.ErrResourceLimit. + return fmt.Errorf( + "XPDY0130: evaluation constructed more than %d result-tree "+ + "nodes; the transformation is building a tree too large to "+ + "hold: %w", MaxNodes, xdm.ErrResourceLimit) +} + +// NodesRefused reports the node-budget refusal if any charge has already +// crossed MaxNodes, and charges nothing. A host calls it where a transform +// finishes: the builders latch a refusal per builder, so a built-in rule or a +// temporary tree that crossed the bound would otherwise end as truncated +// output with no error. +func (c *Context) NodesRefused() error { + if c == nil || c.ev().nodes == nil || atomic.LoadInt64(c.ev().nodes) <= MaxNodes { + return nil + } + return nodesRefusal() +} + // ChargeNodes charges n constructed result-tree nodes against the evaluation // budget, reporting XPDY0130 when the budget is exhausted. // diff --git a/xslt/nodebudget_test.go b/xslt/nodebudget_test.go index b6738c94..02b43c14 100644 --- a/xslt/nodebudget_test.go +++ b/xslt/nodebudget_test.go @@ -269,3 +269,28 @@ func TestRepeatedCopyOfIsRefused(t *testing.T) { _, err = st.Transform(context.Background(), doc.Root, TransformOptions{}) wantNodeRefusal(t, err) } + +// The built-in shallow-copy rule writes through a builder whose latched +// refusal only execSequenceBody polls, so a mode with nothing but +// on-no-match="shallow-copy" used to stop building at the bound and return the +// rest as empty elements, exiting 0. 700,000 t is 2.1 million +// nodes, over MaxNodes. +func TestBuiltInRuleResultTreeOverrunIsRefused(t *testing.T) { + sd, err := xdm.ParseString(``+ + ``, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + st, err := Compile(sd.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + doc, err := xdm.ParseString(""+strings.Repeat(`t`, 700000)+"", + xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + _, err = st.Transform(context.Background(), doc.Root, TransformOptions{}) + wantNodeRefusal(t, err) +} diff --git a/xslt/transform.go b/xslt/transform.go index 1c63e8df..994d45f2 100644 --- a/xslt/transform.go +++ b/xslt/transform.go @@ -748,6 +748,14 @@ func (s *Stylesheet) Transform(ctx context.Context, source *xdm.Node, opts Trans } } + // A budget refusal is latched by the builder that met it and is only + // reported by execSequenceBody's loop. The built-in rules, and any + // temporary tree, write through a builder nobody polls, so output cut + // short at the bound would otherwise be returned as a success. The budget + // counter is the transform's, so it sees the overrun wherever it happened. + if err := rt.ctx.NodesRefused(); err != nil { + return nil, err + } nodes := out.Sequence() if haveRaw { nodes = rawResult From e1395259a63f0e9031c2bf0fd807ce3a29d0adbf Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:12:45 -0400 Subject: [PATCH 41/98] xdm: skip the XInclude copy when the tree has no xi element --- xdm/xinclude.go | 111 +++++++++++++++++++++--------------- xdm/xinclude_nocopy_test.go | 43 ++++++++++++++ 2 files changed, 108 insertions(+), 46 deletions(-) create mode 100644 xdm/xinclude_nocopy_test.go diff --git a/xdm/xinclude.go b/xdm/xinclude.go index 3107308c..40132552 100644 --- a/xdm/xinclude.go +++ b/xdm/xinclude.go @@ -113,6 +113,10 @@ func ProcessXInclude(tree *Tree, opts XIncludeOptions) (*Tree, error) { if base == "" { base = tree.Root.DocumentURI() } + if !mentionsXInclude(tree) { + // Nothing to replace: the copy processTree would make is the tree. + return tree, nil + } p := &includeProc{opts: opts, budget: &entityBudget{}} if base != "" { p.stack = append(p.stack, base) @@ -161,6 +165,18 @@ type includeProc struct { done map[*Node]*Node } +// mentionsXInclude reports whether t could hold an XInclude-namespace element. +// Every element name is in the name table, so a table without the namespace +// proves the tree has no xi:include or xi:fallback to process. +func mentionsXInclude(t *Tree) bool { + for _, q := range t.names { + if q.URI == NSXInclude { + return true + } + } + return false +} + // processTree builds the included form of t: a new tree with every xi:include // replaced by what it includes. func (p *includeProc) processTree(t *Tree, base string, depth int) (*Tree, error) { @@ -255,18 +271,10 @@ func appendLeafKeep(dst, n *Node) *Node { // include appends to dst what one xi:include element includes. // -// The inclusion is built in a holder first and only appended once it has -// succeeded: a failure anywhere in it must leave dst as it was, so that the -// fallback can take its place. +// An inclusion is appended only once it has succeeded: a failure anywhere in +// it must leave dst as it was, so that the fallback can take its place. func (p *includeProc) include(dst, inc *Node, depth int) error { - holder, err := p.expandInclude(inc, depth) - if err != nil { - return err - } - for n := range holder.Children() { - appendSubtreeKeep(dst, n) - } - return nil + return p.expandInclude(dst, inc, depth) } // appendSubtreeKeep appends a deep copy of n, merging text as appendLeafKeep @@ -287,11 +295,10 @@ func appendSubtreeKeep(dst, n *Node) { // positions of what it holds survive the move. func newHolder() *Node { return NewTree().Root } -// expandInclude computes, in a holder, the replacement for one xi:include -// element. -func (p *includeProc) expandInclude(inc *Node, depth int) (*Node, error) { +// expandInclude appends to dst the replacement for one xi:include element. +func (p *includeProc) expandInclude(dst, inc *Node, depth int) error { if depth >= maxIncludeDepth { - return nil, fatalInclude{fmt.Errorf( + return fatalInclude{fmt.Errorf( "resource limit exceeded: xi:include nesting exceeds %d levels: %w", maxIncludeDepth, ErrResourceLimit)} } @@ -306,20 +313,20 @@ func (p *includeProc) expandInclude(inc *Node, depth int) (*Node, error) { if c.Name().Local != "fallback" { // Section 3.1: the content of xi:include is "(fallback?)", // so any other XInclude-namespace child is a fatal error. - return nil, fmt.Errorf( + return fmt.Errorf( "xi:%s is not permitted as a child of xi:include", c.Name().Local) } seen++ } } if seen > 1 { - return nil, fmt.Errorf("xi:include has more than one xi:fallback") + return fmt.Errorf("xi:include has more than one xi:fallback") } href := inc.AttrValue("href") if href != "" { if err := ValidateXIncludeHref(href); err != nil { - return nil, err + return err } } parse := inc.AttrValue("parse") @@ -332,7 +339,7 @@ func (p *includeProc) expandInclude(inc *Node, depth int) (*Node, error) { // something to fall back from: the attribute is the processor's // instruction, not the resource's condition, so a fallback would be // answering the wrong question. - return nil, fmt.Errorf("xi:include parse=%q must be \"xml\" or \"text\"", parse) + return fmt.Errorf("xi:include parse=%q must be \"xml\" or \"text\"", parse) } xptr := inc.AttrValue("xpointer") @@ -389,10 +396,17 @@ func (p *includeProc) expandInclude(inc *Node, depth int) (*Node, error) { // the loop rule does not apply — a text inclusion cannot recurse. holder, err = p.fetchText(target, base, encoding) default: - holder, err = p.fetch(target, base, parse, xptr, encoding, depth) + // fetch appends to dst itself, and only once it cannot fail. + err = p.fetch(dst, target, base, parse, xptr, encoding, depth) + if err == nil { + return nil + } } - if err == nil { - return holder, nil + if holder != nil && err == nil { + for n := range holder.Children() { + appendSubtreeKeep(dst, n) + } + return nil } // Section 4.3: "If the resource ... cannot be fetched ... the processor @@ -406,7 +420,7 @@ func (p *includeProc) expandInclude(inc *Node, depth int) (*Node, error) { // a document could quietly get a different result by being expensive, and // a fallback chain would become a way to keep asking after being told no. if fatalIncludeError(err) { - return nil, err + return err } if fb := fallbackOf(inc); fb != nil { // The fallback's own content may itself contain xi:include elements @@ -415,14 +429,10 @@ func (p *includeProc) expandInclude(inc *Node, depth int) (*Node, error) { // like any other content. The depth is carried through so that a // fallback chain cannot be used to sidestep the nesting bound. fbBase := elementBase(fb) - holder := newHolder() - if err := p.copyChildren(holder, fb, fbBase, depth+1); err != nil { - return nil, err - } - return holder, nil + return p.copyChildren(dst, fb, fbBase, depth+1) } // Section 4.3: "if the fallback element is absent, it is a fatal error." - return nil, fmt.Errorf("xi:include of %q failed and has no xi:fallback: %w", target, err) + return fmt.Errorf("xi:include of %q failed and has no xi:fallback: %w", target, err) } // selectLocal resolves an xpointer against the including document, for an @@ -540,25 +550,32 @@ func copySubtree(dst, n *Node, base string) { } } -// fetch reads one resource and returns, in a holder, the nodes that replace -// the xi:include element. -func (p *includeProc) fetch(target, base, parse, xptr, encoding string, depth int) (*Node, error) { +// fetch reads one resource and appends to dst the nodes that replace the +// xi:include element. It appends only once nothing can fail any more. +func (p *includeProc) fetch(dst *Node, target, base, parse, xptr, encoding string, depth int) error { if parse == "text" { - return p.fetchText(target, base, encoding) + holder, err := p.fetchText(target, base, encoding) + if err != nil { + return err + } + for n := range holder.Children() { + appendSubtreeKeep(dst, n) + } + return nil } if p.fetches >= maxIncludeFetches { - return nil, fatalInclude{fmt.Errorf( + return fatalInclude{fmt.Errorf( "resource limit exceeded: document performs more than %d inclusions: %w", maxIncludeFetches, ErrResourceLimit)} } if p.opts.Resolver == nil { - return nil, fmt.Errorf("no include resolver: inclusions are not permitted") + return fmt.Errorf("no include resolver: inclusions are not permitted") } p.fetches++ data, uri, err := p.opts.Resolver.ResolveInclude(target, base, encoding) if err != nil { - return nil, err + return err } // A cycle is checked on the URI the resolver reports rather than on the @@ -567,7 +584,7 @@ func (p *includeProc) fetch(target, base, parse, xptr, encoding string, depth in // without this the recursion below simply does not terminate. for _, u := range p.stack { if u == uri { - return nil, fatalInclude{fmt.Errorf( + return fatalInclude{fmt.Errorf( "circular xi:include loop: %q includes itself", uri)} } } @@ -595,19 +612,22 @@ func (p *includeProc) fetch(target, base, parse, xptr, encoding string, depth in // entity bomb expanded 6,291,456 bytes — six times the ceiling — and // the document was accepted, the refusal swallowed at every level. if errors.Is(err, ErrResourceLimit) { - return nil, fatalInclude{err} + return fatalInclude{err} } - return nil, err + return err } // Process the included document before selecting, so that an xpointer // may address content that an inner inclusion brought in. The stack // grows for the duration. p.stack = append(p.stack, uri) - done, err := p.processTree(sub, uri, depth+1) + done := sub + if mentionsXInclude(sub) { + done, err = p.processTree(sub, uri, depth+1) + } p.stack = p.stack[:len(p.stack)-1] if err != nil { - return nil, err + return err } var picked []*Node @@ -624,22 +644,21 @@ func (p *includeProc) fetch(target, base, parse, xptr, encoding string, depth in } else { picked, err = selectXPointer(done.Root, xptr) if err != nil { - return nil, err + return err } if len(picked) == 0 { // Section 4.4: an xpointer that "does not identify any // resource" is an error, recoverable through fallback — which // is what returning an error here achieves, since the caller // consults xi:fallback on any error from this function. - return nil, fmt.Errorf("xpointer %q selected nothing in %s", xptr, uri) + return fmt.Errorf("xpointer %q selected nothing in %s", xptr, uri) } } - holder := newHolder() for _, n := range picked { - copyFixedUp(holder, n, base) + copyFixedUp(dst, n, base) } - return holder, nil + return nil } // copyFixedUp appends a copy of n, a top-level included node, with the base diff --git a/xdm/xinclude_nocopy_test.go b/xdm/xinclude_nocopy_test.go new file mode 100644 index 00000000..dd25c3d6 --- /dev/null +++ b/xdm/xinclude_nocopy_test.go @@ -0,0 +1,43 @@ +package xdm + +import ( + "runtime" + "strings" + "testing" +) + +// A tree with no XInclude element has nothing to process, so neither the +// including document nor an included one is copied (-xinclude +// on a 100 MB document peaked 500 MB higher for no inclusion at all). +func TestXIncludeSkipsCopyWithoutXIElements(t *testing.T) { + tree := parseWithBase(t, "mem:/a.xml", "text") + out, err := ProcessXInclude(tree, XIncludeOptions{}) + if err != nil || out != tree { + t.Fatalf("plain tree: got %p (tree %p), err %v; want the tree itself", out, tree, err) + } + + big := "" + strings.Repeat("text", 20000) + "" + parseBytes := allocBytes(func() { parseWithBase(t, "mem:/big.xml", big) }) + doc := parseWithBase(t, "mem:/a.xml", ``) + res := &mapResolver{files: map[string]string{"mem:/big.xml": big}} + incBytes := allocBytes(func() { + if _, err := ProcessXInclude(doc, XIncludeOptions{Resolver: res}); err != nil { + t.Fatal(err) + } + }) + // Parse plus the one copy into the result is about 2.2 parses; a copy of + // the included document before it, or through a holder, adds 1 each. + if incBytes > parseBytes*13/5 { + t.Errorf("including a plain document allocated %d bytes; parsing it alone is %d", incBytes, parseBytes) + } + t.Logf("parse %d include %d", parseBytes, incBytes) +} + +func allocBytes(f func()) uint64 { + var a, b runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&a) + f() + runtime.ReadMemStats(&b) + return b.TotalAlloc - a.TotalAlloc +} From 21995a6615b4a9d72d5d92fd7f721e7dcc4f50a7 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:13:05 -0400 Subject: [PATCH 42/98] xslt: charge retained xsl:key entries and group-by keys to the item budget --- xpath/context.go | 33 +++++++++++++++++++++++------ xslt/grouping.go | 7 ++++++ xslt/nodebudget_test.go | 47 +++++++++++++++++++++++++++++++++++++++++ xslt/rtfuncs.go | 27 +++++++++++++++++------ 4 files changed, 101 insertions(+), 13 deletions(-) diff --git a/xpath/context.go b/xpath/context.go index 4ccd67e0..9627a773 100644 --- a/xpath/context.go +++ b/xpath/context.go @@ -1333,13 +1333,32 @@ func (c *Context) countItems(n int) error { return nil } if lim := c.itemLimit(); atomic.AddInt64(c.ev().items, int64(n)) > lim { - // The code is kept -- the suites and callers read it -- and the - // sentinel added, because this is the processor declining to - // allocate rather than anything wrong with the expression. - return fmt.Errorf( - "XPDY0130: evaluation materialised more than %d items; "+ - "the expression is building a sequence too large to hold: %w", - lim, xdm.ErrResourceLimit) + return itemsRefusal(lim) + } + return nil +} + +// itemsRefusal is the XPDY0130 the item budget raises. +func itemsRefusal(lim int64) error { + // The code is kept -- the suites and callers read it -- and the + // sentinel added, because this is the processor declining to + // allocate rather than anything wrong with the expression. + return fmt.Errorf( + "XPDY0130: evaluation materialised more than %d items; "+ + "the expression is building a sequence too large to hold: %w", + lim, xdm.ErrResourceLimit) +} + +// ItemsRefused reports the item-budget refusal if total exceeds the item bound +// in force, and charges nothing. +// +// A host that retains items across many expression evaluations -- xsl:key's +// index, for-each-group's groups -- keeps the running total itself, because +// Compiled.Eval resets the shared counter for every expression and a charge +// made through ChargeItems would be cleared by the next node's evaluation. +func (c *Context) ItemsRefused(total int64) error { + if lim := c.itemLimit(); total > lim { + return itemsRefusal(lim) } return nil } diff --git a/xslt/grouping.go b/xslt/grouping.go index 89937151..c4e822d9 100644 --- a/xslt/grouping.go +++ b/xslt/grouping.go @@ -258,6 +258,9 @@ func groupByKey(rt *runtime, seq xdm.Sequence, key *xpath.Compiled, // when a grouping actually mixes numeric types. var numericGroups []int numericTypes := map[xdm.TypeCode]bool{} + // Eval resets the item counter per expression, so what the groups retain + // is totalled here: one entry per key value an item joins. + var held int64 for idx, it := range seq { sub := rt.withFocus(it, idx+1, len(seq)) @@ -265,6 +268,10 @@ func groupByKey(rt *runtime, seq xdm.Sequence, key *xpath.Compiled, if err != nil { return nil, err } + held += int64(len(vals)) + if err := rt.ctx.ItemsRefused(held); err != nil { + return nil, err + } // Section 14.2 forms the groups from the *distinct* key values of an // item, so an item whose key expression yields the same value twice // still joins that group once. group-by="number(@pop), diff --git a/xslt/nodebudget_test.go b/xslt/nodebudget_test.go index 02b43c14..725d25d9 100644 --- a/xslt/nodebudget_test.go +++ b/xslt/nodebudget_test.go @@ -294,3 +294,50 @@ func TestBuiltInRuleResultTreeOverrunIsRefused(t *testing.T) { _, err = st.Transform(context.Background(), doc.Root, TransformOptions{}) wantNodeRefusal(t, err) } + +// runRetainedKeys runs body over crossProductDoc, with a key or group-by of +// 1,000 values per node. 5,100 nodes retain 5.1 million entries, just past the +// 5 million item bound; no single expression evaluation comes near it, which +// is why the per-expression counter never saw it. +func runRetainedKeys(t *testing.T, body string) error { + t.Helper() + sd, err := xdm.ParseString(``+body+ + ``, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + st, err := Compile(sd.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + doc, err := xdm.ParseString(crossProductDoc(5100), xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + _, err = st.Transform(context.Background(), doc.Root, TransformOptions{}) + return err +} + +func wantItemRefusal(t *testing.T, err error) { + t.Helper() + if err == nil { + t.Fatal("expected XPDY0130; the retained keys were not charged") + } + if !errors.Is(err, xdm.ErrResourceLimit) || + !strings.Contains(err.Error(), "materialised more than") { + t.Errorf("not the item-budget refusal: %v", err) + } +} + +func TestKeyIndexEntriesAreChargedToTheItemBudget(t *testing.T) { + wantItemRefusal(t, runRetainedKeys(t, + ``+ + ``)) +} + +func TestGroupByKeysAreChargedToTheItemBudget(t *testing.T) { + wantItemRefusal(t, runRetainedKeys(t, + ``+ + ``)) +} diff --git a/xslt/rtfuncs.go b/xslt/rtfuncs.go index 15bba1e9..30fa3c4d 100644 --- a/xslt/rtfuncs.go +++ b/xslt/rtfuncs.go @@ -1320,6 +1320,21 @@ func (rt *runtime) keyIndexFor(name string, defs []*keyDef, root *xdm.Node, } idx := map[string]xdm.Sequence{} + // Every key entry is retained until the transform ends, and Eval resets + // the item counter per expression, so the running total is kept here + // and checked against the item bound. use="1 to 1000" over 100,000 nodes + // was a hundred million entries and 3.2 GB. + var held int64 + addKeys := func(ks []string, n *xdm.Node) error { + held += int64(len(ks)) + if err := ctx.ItemsRefused(held); err != nil { + return err + } + for _, k := range ks { + idx[k] = append(idx[k], n) + } + return nil + } // A pattern that fails because it reads a global still being computed is // a circularity, not a node-specific failure, and Matches has already // recovered from it by the time it returns -- so the check is made here, @@ -1359,8 +1374,8 @@ func (rt *runtime) keyIndexFor(name string, defs []*keyDef, root *xdm.Node, if err != nil { return err } - for _, k := range ks { - idx[k] = append(idx[k], n) + if err := addKeys(ks, n); err != nil { + return err } } // Attributes can be key targets, so they are visited too. @@ -1385,8 +1400,8 @@ func (rt *runtime) keyIndexFor(name string, defs []*keyDef, root *xdm.Node, if err != nil { return err } - for _, k := range ks { - idx[k] = append(idx[k], a) + if err := addKeys(ks, a); err != nil { + return err } } } @@ -1417,8 +1432,8 @@ func (rt *runtime) keyIndexFor(name string, defs []*keyDef, root *xdm.Node, if err != nil { return err } - for _, k := range ks { - idx[k] = append(idx[k], nsNode) + if err := addKeys(ks, nsNode); err != nil { + return err } } } From 3b7150152f3d676dd8527571f4b844df32eacbdd Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:13:24 -0400 Subject: [PATCH 43/98] xdm: clip the TrackPositions line index at the end of the parse --- xdm/lineindex_clip_test.go | 30 ++++++++++++++++++++++++++++++ xdm/parse.go | 5 +++++ 2 files changed, 35 insertions(+) create mode 100644 xdm/lineindex_clip_test.go diff --git a/xdm/lineindex_clip_test.go b/xdm/lineindex_clip_test.go new file mode 100644 index 00000000..ad3d42c4 --- /dev/null +++ b/xdm/lineindex_clip_test.go @@ -0,0 +1,30 @@ +package xdm + +import ( + "strings" + "testing" +) + +// The line index is built by append, which leaves its slack allocated; a +// parse must hand back an index with none (TrackPositions on a 100 MB +// document kept 0.9 MB unused). +func TestTrackPositionsLineIndexHasNoSlack(t *testing.T) { + for _, n := range []int{1000, 5000} { // 5000 lands mid-doubling + src := "" + strings.Repeat("\n", n) + "" + opts := ParseOptions{TrackPositions: true} + fromString, err := ParseString(src, opts) + if err != nil { + t.Fatal(err) + } + fromReader, err := Parse(strings.NewReader(src), opts) + if err != nil { + t.Fatal(err) + } + for _, tree := range []*Tree{fromString, fromReader} { + l := tree.lineIdx() + if len(l.starts) < n || cap(l.starts) != len(l.starts) { + t.Errorf("n=%d: line index len %d cap %d", n, len(l.starts), cap(l.starts)) + } + } + } +} diff --git a/xdm/parse.go b/xdm/parse.go index 702184f3..950f861b 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -729,6 +729,11 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { } else { lines = tee.lines } + // append's doubling leaves up to a third of the index unused, kept + // for the life of the tree. + if cap(lines.starts) > len(lines.starts) { + lines.starts = append(make([]int32, 0, len(lines.starts)), lines.starts...) + } tree.ownSource().lines = lines } tree.XMLVersion = "1.0" From de4a515e42dde312e6502a92085d183fb0348f44 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:14:29 -0400 Subject: [PATCH 44/98] xslt: bound the FileResolver document cache by source bytes as well as entries --- xslt/resolver.go | 53 +++++++++++++++++++++++++++---------- xslt/resolver_cache_test.go | 42 ++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 15 deletions(-) diff --git a/xslt/resolver.go b/xslt/resolver.go index 220300aa..d76c95e8 100644 --- a/xslt/resolver.go +++ b/xslt/resolver.go @@ -92,6 +92,11 @@ type FileResolver struct { mu sync.Mutex cache map[string]*xdm.Tree + // sizes is the source size charged for each cache entry, and cacheBytes + // their sum, both under mu. A Preload entry is charged nothing: the host + // parsed it and holds it already. + sizes map[string]int64 + cacheBytes int64 // inflight tracks a path being read and parsed right now, so that // concurrent callers wait for that one parse rather than each doing their // own. Node identity requires it: fn:doc is defined to return the *same* @@ -156,6 +161,17 @@ type loadCall struct { // run never reaches it and cheap enough that reaching it costs one reparse. const resolverCacheMax = 256 +// resolverCacheMaxBytes bounds the source bytes of the documents a resolver +// retains: 32 MB, charged by file size. +// +// The entry count alone does not bound memory, because a document is not a +// fixed size: 256 documents of 158 KB retained 201 MB, about five times their +// file size once parsed. 32 MB of source is therefore on the order of 150 MB +// of trees, and a working set of ordinary stylesheets and code lists is far +// below it. A document that alone exceeds the bound is still cached, after +// everything else has been evicted; MaxBytes is what refuses a file outright. +const resolverCacheMaxBytes int64 = 32 << 20 + // DefaultMaxResourceBytes bounds one file read through a FileResolver when // MaxBytes is zero: 64 MB. // @@ -487,7 +503,7 @@ func (r *FileResolver) Preload(uri string, tree *xdm.Tree) { // stylesheet can reach, so an unbounded Preload was an invariant violation // and not an attack surface -- but a host that preloads a document per // request would still have grown the map for the life of the process. - r.publish(path, tree) + r.publish(path, tree, 0) } // load parses a file, caching the result. @@ -564,11 +580,11 @@ func (r *FileResolver) loadTracked( r.inflight[key] = call r.mu.Unlock() - tree, err := r.parseUncached(path, trackPos, b) + tree, size, err := r.parseUncached(path, trackPos, b) r.mu.Lock() if err == nil { - r.publish(path, tree) + r.publish(path, tree, size) } delete(r.inflight, key) r.mu.Unlock() @@ -580,7 +596,7 @@ func (r *FileResolver) loadTracked( } // publish stores a parsed tree. The caller holds r.mu. -func (r *FileResolver) publish(path string, tree *xdm.Tree) { +func (r *FileResolver) publish(path string, tree *xdm.Tree, size int64) { // The cache is created on first use. A FileResolver is documented as // usable as a plain literal — &FileResolver{Roots: ...} — so a nil map // here is the ordinary case, not a caller's mistake, and writing to one @@ -606,27 +622,36 @@ func (r *FileResolver) publish(path string, tree *xdm.Tree) { // and still retains a working set, whereas the recency metadata an LRU // wants would have to be written on every cache HIT. A false eviction // costs one reparse, which is what a miss cost before. - for len(r.cache) >= resolverCacheMax { - if _, ok := r.cache[path]; ok { - // Replacing an entry that is already present does not grow the - // map, so nothing has to be evicted for it. - break - } + if r.sizes == nil { + r.sizes = map[string]int64{} + } + // A replaced entry is removed first and gives its charge back, so a + // replacement never counts against the bounds twice. + r.cacheBytes -= r.sizes[path] + delete(r.sizes, path) + delete(r.cache, path) + for len(r.cache) > 0 && + (len(r.cache) >= resolverCacheMax || + r.cacheBytes+size > resolverCacheMaxBytes) { for victim := range r.cache { delete(r.cache, victim) + r.cacheBytes -= r.sizes[victim] + delete(r.sizes, victim) break } } r.cache[path] = tree + r.sizes[path] = size + r.cacheBytes += size } // parseUncached reads and parses a file. It holds no lock: this is the work // that used to run under r.mu. func (r *FileResolver) parseUncached( - path string, trackPos bool, b *xdm.EntityBudget) (*xdm.Tree, error) { + path string, trackPos bool, b *xdm.EntityBudget) (*xdm.Tree, int64, error) { data, err := r.readConfined(path) if err != nil { - return nil, err + return nil, 0, err } // A file: URI rather than the bare path. This base URI is stamped on // every element of the module and is what fn:document, fn:resolve-uri @@ -655,9 +680,9 @@ func (r *FileResolver) parseUncached( // in place. See ResolveModuleWith. tree, err := xdm.ParseString(string(data), opts.WithEntityBudget(b)) if err != nil { - return nil, fmt.Errorf("parsing %s: %w", path, err) + return nil, 0, fmt.Errorf("parsing %s: %w", path, err) } - return tree, nil + return tree, int64(len(data)), nil } // fileURIToPath turns a file: URI back into a filesystem path, and leaves diff --git a/xslt/resolver_cache_test.go b/xslt/resolver_cache_test.go index ca582096..eab6a8e4 100644 --- a/xslt/resolver_cache_test.go +++ b/xslt/resolver_cache_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "github.com/knroy/go-xml/v2/xdm" @@ -165,7 +166,7 @@ func TestResolverCacheReplaceDoesNotEvict(t *testing.T) { anyPath, anyTree = k, v break } - res.publish(anyPath, anyTree) + res.publish(anyPath, anyTree, 0) got := len(res.cache) res.mu.Unlock() @@ -178,3 +179,42 @@ func TestResolverCacheReplaceDoesNotEvict(t *testing.T) { full, got) } } + +// The entry count is not a memory bound: documents differ in size, and 256 of +// 158 KB retained 201 MB. 40 documents of 1 MB are far under resolverCacheMax +// entries and over resolverCacheMaxBytes, so only the byte bound can evict. +func TestResolverCacheIsBoundedByBytes(t *testing.T) { + dir := t.TempDir() + res, err := NewFileResolver(dir) + if err != nil { + t.Fatal(err) + } + body := "" + strings.Repeat("x", 1<<20) + "" + for i := 0; i < 40; i++ { + name := filepath.Join(dir, fmt.Sprintf("big%d.xml", i)) + if err := os.WriteFile(name, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if _, err := res.ResolveDocument(name, ""); err != nil { + t.Fatal(err) + } + } + res.mu.Lock() + defer res.mu.Unlock() + // The documented 32 MB, spelled out so that changing the constant cannot + // quietly change what this pins. + if res.cacheBytes > 32<<20 { + t.Errorf("cache charges %d bytes, over the 32 MB bound", res.cacheBytes) + } + var sum int64 + for _, n := range res.sizes { + sum += n + } + if sum != res.cacheBytes || len(res.sizes) != len(res.cache) { + t.Errorf("accounting drifted: sizes sum %d (%d entries), cacheBytes %d, cache %d entries", + sum, len(res.sizes), res.cacheBytes, len(res.cache)) + } + if len(res.cache) < 20 { + t.Errorf("only %d documents retained; the bound evicted far more than it needed", len(res.cache)) + } +} From 9f0db9ccf9405a9198a1e07b2938c43bedae9c84 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:14:52 -0400 Subject: [PATCH 45/98] relaxng: publish nameMemo stores as amortised snapshots instead of copying the map each time --- relaxng/namememo_test.go | 34 ++++++++++++++++++++++++++ relaxng/pattern.go | 53 ++++++++++++++++++++++++++++++---------- 2 files changed, 74 insertions(+), 13 deletions(-) create mode 100644 relaxng/namememo_test.go diff --git a/relaxng/namememo_test.go b/relaxng/namememo_test.go new file mode 100644 index 00000000..d68b033e --- /dev/null +++ b/relaxng/namememo_test.go @@ -0,0 +1,34 @@ +package relaxng + +import ( + "fmt" + "runtime" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// store used to copy the whole map every time: 1000 stores allocated tens of +// megabytes. Now the copies sum to O(n). Concurrent readers must stay safe +// (run under -race) and the cap must hold. +func TestNameMemoStoreAmortised(t *testing.T) { + var c nameMemo + var m0, m1 runtime.MemStats + runtime.ReadMemStats(&m0) + for i := 0; i < maxOpenMemo; i++ { + c.store(xdm.QName{Local: fmt.Sprintf("n%d", i)}, emptyPat{}) + } + runtime.ReadMemStats(&m1) + if got := m1.TotalAlloc - m0.TotalAlloc; got > 2<<20 { + t.Errorf("%d stores allocated %d KB, want under 2048 KB", maxOpenMemo, got>>10) + } + if _, ok := c.load(xdm.QName{Local: "n0"}); !ok { + t.Error("first entry not published") + } + c.store(xdm.QName{Local: "over"}, emptyPat{}) + c.store(xdm.QName{Local: "over2"}, emptyPat{}) + n := len(*c.m.Load()) + len(c.pending) + if n > maxOpenMemo { + t.Errorf("%d entries, cap is %d", n, maxOpenMemo) + } +} diff --git a/relaxng/pattern.go b/relaxng/pattern.go index f4510ecf..67894487 100644 --- a/relaxng/pattern.go +++ b/relaxng/pattern.go @@ -175,22 +175,32 @@ type staticInfo struct { // nameMemo is a copy-on-write map from a name to a pattern, shared by every // validation of a schema: a lookup is one atomic load and a map read with no -// boxing, and a store copies the map under a lock. The entries are capped at +// boxing, and a store lands in a small pending map under a lock and is published as a +// new snapshot once the pending entries outgrow it. The entries are capped at // maxOpenMemo so that a document cannot grow it without bound by inventing // names, which also bounds each copy. type nameMemo struct { - m atomic.Pointer[map[nameKey]pattern] - mu sync.Mutex + m atomic.Pointer[map[nameKey]pattern] + mu sync.Mutex + pending map[nameKey]pattern // stored but not yet published in m; under mu + np atomic.Int32 // len(pending), so a load skips the lock when empty } type nameKey struct{ ns, local string } func (c *nameMemo) load(n xdm.QName) (pattern, bool) { - m := c.m.Load() - if m == nil { + k := nameKey{n.URI, n.Local} + if m := c.m.Load(); m != nil { + if p, ok := (*m)[k]; ok { + return p, true + } + } + if c.np.Load() == 0 { return nil, false } - p, ok := (*m)[nameKey{n.URI, n.Local}] + c.mu.Lock() + p, ok := c.pending[k] + c.mu.Unlock() return p, ok } @@ -201,15 +211,32 @@ func (c *nameMemo) store(n xdm.QName, p pattern) { if o := c.m.Load(); o != nil { old = *o } - if len(old) >= maxOpenMemo { - return + k := nameKey{n.URI, n.Local} + if _, ok := c.pending[k]; !ok { + if len(old)+len(c.pending) >= maxOpenMemo { + return + } } - m := make(map[nameKey]pattern, len(old)+1) - for k, v := range old { - m[k] = v + if c.pending == nil { + c.pending = make(map[nameKey]pattern) + } + c.pending[k] = p + c.np.Store(int32(len(c.pending))) + // Publish once the pending entries outgrow the snapshot, so the copies + // sum to O(n) over n stores. A load that misses the snapshot checks the + // pending map. + if len(c.pending) > len(old) { + m := make(map[nameKey]pattern, len(old)+len(c.pending)) + for k, v := range old { + m[k] = v + } + for k, v := range c.pending { + m[k] = v + } + c.m.Store(&m) + c.pending = nil + c.np.Store(0) } - m[nameKey{n.URI, n.Local}] = p - c.m.Store(&m) } type patBox struct { From 5dc120c803cc7f73dbe780dbf8bcd5414df73052 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:14:52 -0400 Subject: [PATCH 46/98] xsd, dtd: dedup small state sets by scan; reuse child-name buffer; skip attribute map with no ATTLIST --- dtd/alloc_perelement_test.go | 35 +++++++++++++++++++++++++++++++++++ dtd/validate.go | 7 ++++++- xsd/nfa.go | 21 ++++++++++++++++++++- xsd/stateset_small_test.go | 25 +++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 dtd/alloc_perelement_test.go create mode 100644 xsd/stateset_small_test.go diff --git a/dtd/alloc_perelement_test.go b/dtd/alloc_perelement_test.go new file mode 100644 index 00000000..c3c58e57 --- /dev/null +++ b/dtd/alloc_perelement_test.go @@ -0,0 +1,35 @@ +package dtd + +import ( + "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// With no ATTLIST for an element, checkAttributes built a map anyway, and the +// child-name slice was rebuilt for every element. +func TestValidateAllocsPerElement(t *testing.T) { + d, err := Parse(` + ]>`) + if err != nil { + t.Fatal(err) + } + const n = 2000 + tree, err := xdm.ParseString(""+strings.Repeat("

", n)+"", xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var m0, m1 runtime.MemStats + runtime.ReadMemStats(&m0) + err = Validate(tree.Root, d, Options{}) + runtime.ReadMemStats(&m1) + if err != nil { + t.Fatal(err) + } + // 6001 elements: 8062 allocations with the fix, 12062 without. + if got := m1.Mallocs - m0.Mallocs; got > 10000 { + t.Errorf("%d allocations for %d elements, want at most 10000", got, 3*n+1) + } +} diff --git a/dtd/validate.go b/dtd/validate.go index 8b946bcb..e95fb948 100644 --- a/dtd/validate.go +++ b/dtd/validate.go @@ -149,6 +149,7 @@ type validator struct { allowUndec bool errs []*Error matchers map[string]*xsd.SequenceMatcher + names []xdm.QName // child-name buffer reused across elements // ids are every ID value seen, for uniqueness and for resolving IDREF. ids map[string]bool // partial records that the DTD was read from the internal subset alone @@ -270,12 +271,13 @@ func (v *validator) checkContent(el *xdm.Node) { v.fail(el, "content model of %s: %v", el.Name().Local, err) return } - var names []xdm.QName + names := v.names[:0] for c := range el.Children() { if c.Kind() == xdm.KindElement { names = append(names, xdm.QName{Local: c.Name().Local}) } } + v.names = names if ok, at := m.Match(names); !ok { if at < len(names) { v.fail(el, "element %s is not permitted here in the "+ @@ -302,6 +304,9 @@ func (v *validator) matcherFor(decl *Element) (*xsd.SequenceMatcher, error) { // checkAttributes applies the ATTLIST declarations for an element. func (v *validator) checkAttributes(el *xdm.Node) { decls := v.dtd.Attributes[el.Name().Local] + if len(decls) == 0 { + return + } present := map[string]string{} for a := range el.Attrs() { name := a.Name().Local diff --git a/xsd/nfa.go b/xsd/nfa.go index 61e9ac52..c5a19ffa 100644 --- a/xsd/nfa.go +++ b/xsd/nfa.go @@ -88,10 +88,29 @@ type stateSet struct { } func newStateSet() *stateSet { - return &stateSet{seen: map[matchState]bool{}} + return &stateSet{} } +// smallSet is the size up to which add deduplicates by scanning order; the +// map is only built for a larger set. +const smallSet = 8 + func (s *stateSet) add(st matchState) { + if s.seen == nil { + for _, o := range s.order { + if o == st { + return + } + } + s.order = append(s.order, st) + if len(s.order) > smallSet { + s.seen = make(map[matchState]bool, 2*smallSet) + for _, o := range s.order { + s.seen[o] = true + } + } + return + } if s.seen[st] { return } diff --git a/xsd/stateset_small_test.go b/xsd/stateset_small_test.go new file mode 100644 index 00000000..0f19ceeb --- /dev/null +++ b/xsd/stateset_small_test.go @@ -0,0 +1,25 @@ +package xsd + +import "testing" + +// A set of up to smallSet states dedups by scanning order and builds no map; +// a larger one switches to a map and must still dedup. +func TestStateSetSmallHasNoMap(t *testing.T) { + s := newStateSet() + for i := 0; i < 2*smallSet; i++ { + s.add(matchState{pos: i % smallSet}) + } + if s.seen != nil { + t.Fatal("a set of smallSet states built a map") + } + if s.len() != smallSet { + t.Fatalf("len = %d, want %d", s.len(), smallSet) + } + for i := 0; i < 3*smallSet; i++ { + s.add(matchState{pos: i}) + s.add(matchState{pos: i}) + } + if s.len() != 3*smallSet { + t.Fatalf("len = %d, want %d", s.len(), 3*smallSet) + } +} From da05b164edbec8b138c398efa4c69fac88e2ef86 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:14:53 -0400 Subject: [PATCH 47/98] xsdnet: share one guarded Transport across fetches --- xsd/xsdnet/guarded_transport_test.go | 11 +++++++++++ xsd/xsdnet/resolve_http.go | 8 +++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 xsd/xsdnet/guarded_transport_test.go diff --git a/xsd/xsdnet/guarded_transport_test.go b/xsd/xsdnet/guarded_transport_test.go new file mode 100644 index 00000000..e51876cc --- /dev/null +++ b/xsd/xsdnet/guarded_transport_test.go @@ -0,0 +1,11 @@ +package xsdnet + +import "testing" + +// Every fetch used to build its own guarded Transport, so no connection was +// reused and each fetch left idle-connection goroutines behind. +func TestGuardedTransportShared(t *testing.T) { + if guardedTransport() != guardedTransport() { + t.Fatal("guardedTransport returned different Transports: fetches cannot reuse connections") + } +} diff --git a/xsd/xsdnet/resolve_http.go b/xsd/xsdnet/resolve_http.go index ef1addf3..d97f2981 100644 --- a/xsd/xsdnet/resolve_http.go +++ b/xsd/xsdnet/resolve_http.go @@ -17,6 +17,7 @@ import ( "net/netip" "net/url" "strings" + "sync" "syscall" "time" @@ -155,7 +156,7 @@ func (r *HTTPResolver) Resolve(namespace, location, base string) (io.ReadCloser, if !r.AllowPrivateAddresses { c := *client if c.Transport == nil { - c.Transport = newGuardedTransport() + c.Transport = guardedTransport() client = &c } } @@ -245,6 +246,11 @@ func (b *limitedBody) Read(p []byte) (int, error) { func (b *limitedBody) Close() error { return b.c.Close() } +// guardedTransport is one shared guarded Transport, so fetches reuse +// connections instead of each building a Transport (and its goroutines). +// The guard holds no per-resolver state. +var guardedTransport = sync.OnceValue(newGuardedTransport) + // newGuardedTransport returns a Transport that refuses to connect to the // address ranges an SSRF is aimed at. // From 967dfb32085e7248f28a12f07a3efc540549c49b Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:15:19 -0400 Subject: [PATCH 48/98] xdm: hold the shared typing names weakly so a dropped schema can be collected --- xdm/recentnames_weak_test.go | 33 +++++++++++++++++++++++++++++++++ xdm/record.go | 31 +++++++++++++++++++++---------- 2 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 xdm/recentnames_weak_test.go diff --git a/xdm/recentnames_weak_test.go b/xdm/recentnames_weak_test.go new file mode 100644 index 00000000..b09d3b7e --- /dev/null +++ b/xdm/recentnames_weak_test.go @@ -0,0 +1,33 @@ +package xdm + +import ( + "runtime" + "testing" + "weak" +) + +// The cache of shared typing names must not keep a TypeEnvironment, and so a +// dropped schema, alive: a process that reloads schemas and validates typed +// copies grew by about 23 MB before the slots were weak. +func TestRecentNamesDoNotPinTypeEnvironment(t *testing.T) { + var nt nodeTyping + env := &TypeEnvironment{} + w := weak.Make(env) + setNames(&nt, typingNames{annotation: "t:one", env: env}) + if nt.get().env != env { + t.Fatal("names lost their environment") + } + // A second set made while the first is in use must still share it. + var other nodeTyping + setNames(&other, typingNames{annotation: "t:one", env: env}) + if other.names != nt.names { + t.Error("equal names in use were not shared") + } + nt, other, env = nodeTyping{}, nodeTyping{}, nil + for i := 0; i < 3 && w.Value() != nil; i++ { + runtime.GC() + } + if w.Value() != nil { + t.Error("TypeEnvironment still reachable after every node dropped its names") + } +} diff --git a/xdm/record.go b/xdm/record.go index 950bae7d..17936515 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -10,6 +10,7 @@ import ( "sync" "sync/atomic" "unsafe" + "weak" ) // The node record. @@ -281,7 +282,12 @@ func (t *nodeTyping) get() *typingNames { // // Each set of names has two slots, so that two in steady use that hash alike // do not keep evicting each other. -var recentNames [1024]atomic.Pointer[typingNames] +// +// The slots are weak: a set of names holds its TypeEnvironment, and a strong +// slot would keep the schema of a process that reloads schemas alive until +// 1024 later sets pushed it out (about 23 MB for a 500-type schema). A set +// no node uses any more is simply made again. +var recentNames [1024]atomic.Pointer[weak.Pointer[typingNames]] // recentSlots is the index of v's first slot in recentNames; the second is // the one after it. @@ -986,20 +992,25 @@ func setNames(t *nodeTyping, v typingNames) { } i := recentSlots(&v) a, b := &recentNames[i], &recentNames[i+1] - pa := a.Load() - if pa != nil && *pa == v { - t.names = pa - return + wa := a.Load() + if wa != nil { + if pa := wa.Value(); pa != nil && *pa == v { + t.names = pa + return + } } - if pb := b.Load(); pb != nil && *pb == v { - t.names = pb - return + if wb := b.Load(); wb != nil { + if pb := wb.Value(); pb != nil && *pb == v { + t.names = pb + return + } } p := new(typingNames) *p = v + w := weak.Make(p) // The newest goes first; the first moves to second. - b.Store(pa) - a.Store(p) + b.Store(wa) + a.Store(&w) t.names = p } From 69414ae0c65ad245a71d1f3b96f17a3f13d70c0d Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:15:54 -0400 Subject: [PATCH 49/98] xslt: build internBindings keys in a reused buffer so a hit does not allocate --- xslt/nodebudget_test.go | 18 ++++++++++++++++++ xslt/stylesheet.go | 24 ++++++++++++++---------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/xslt/nodebudget_test.go b/xslt/nodebudget_test.go index 725d25d9..86e90072 100644 --- a/xslt/nodebudget_test.go +++ b/xslt/nodebudget_test.go @@ -341,3 +341,21 @@ func TestGroupByKeysAreChargedToTheItemBudget(t *testing.T) { ``+ ``)) } + +// internBindings is called for every literal element while compiling, and a +// hit must not allocate: building the key in a strings.Builder cost one +// allocation per call and raised CEN's compile allocation by 12-14%. +func TestInternBindingsHitDoesNotAllocate(t *testing.T) { + setSharedNS(map[*xdm.Node]map[string]string{}) + defer setSharedNS(nil) + bs := []nsBinding{{prefix: "svrl", uri: "http://purl.oclc.org/dsdl/svrl"}, {prefix: "xs", uri: xdm.NSXS}} + first := internBindings(bs) + same := append([]nsBinding(nil), bs...) + if n := testing.AllocsPerRun(100, func() { + if got := internBindings(same); &got[0] != &first[0] { + t.Fatal("equal bindings were not interned to one list") + } + }); n != 0 { + t.Errorf("an interning hit allocates %v times, want 0", n) + } +} diff --git a/xslt/stylesheet.go b/xslt/stylesheet.go index f46d1d60..dfb10eab 100644 --- a/xslt/stylesheet.go +++ b/xslt/stylesheet.go @@ -717,6 +717,7 @@ var sharedNS struct { byContent map[string]map[string]string lists map[string][]nsBinding resolvers map[nsResolverKey]*nsResolver + keyBuf []byte // scratch for internBindings, under the lock } func setSharedNS(m map[*xdm.Node]map[string]string) { @@ -737,23 +738,26 @@ func internBindings(s []nsBinding) []nsBinding { if len(s) == 0 { return s } - var b strings.Builder - for _, ns := range s { - b.WriteString(ns.prefix) - b.WriteByte(0) - b.WriteString(ns.uri) - b.WriteByte(0) - } sharedNS.Lock() defer sharedNS.Unlock() if sharedNS.lists == nil { return s } - k := b.String() - if v, ok := sharedNS.lists[k]; ok { + // Built in a buffer reused across calls and looked up as m[string(b)], + // which does not allocate: a strings.Builder allocated a key per call + // and raised CEN's compile allocation by 12-14%. Only a miss copies it. + b := sharedNS.keyBuf[:0] + for _, ns := range s { + b = append(b, ns.prefix...) + b = append(b, 0) + b = append(b, ns.uri...) + b = append(b, 0) + } + sharedNS.keyBuf = b + if v, ok := sharedNS.lists[string(b)]; ok { return v } - sharedNS.lists[k] = s + sharedNS.lists[string(b)] = s return s } From 887d5d3a0ddc5269cd586ab8c26b383249f51770 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:15:54 -0400 Subject: [PATCH 50/98] xslt: do not keep an oversized function-call key buffer for the whole transform --- xslt/apply.go | 11 ++++++++++- xslt/nodebudget_test.go | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/xslt/apply.go b/xslt/apply.go index 3ca74c57..66eecfd4 100644 --- a/xslt/apply.go +++ b/xslt/apply.go @@ -859,6 +859,9 @@ func (f *userFunction) call(ctx *xpath.Context, args []xdm.Sequence) (xdm.Sequen return res, nil } +// maxKeyBufCap is the largest scratch buffer functionCallKey keeps: 64 KB. +const maxKeyBufCap = 64 << 10 + // functionCallKey identifies one call of a deterministic function by its // arguments. // @@ -896,7 +899,13 @@ func (rt *runtime) functionCallKey(f *userFunction, args []xdm.Sequence) ([]byte return nil, err } } - rt.keyBuf = b + // Kept for the next call only while small: one call with a large argument + // would otherwise leave its buffer for the rest of the transform. + if cap(b) <= maxKeyBufCap { + rt.keyBuf = b + } else { + rt.keyBuf = nil + } return b, nil } diff --git a/xslt/nodebudget_test.go b/xslt/nodebudget_test.go index 86e90072..e9085920 100644 --- a/xslt/nodebudget_test.go +++ b/xslt/nodebudget_test.go @@ -342,6 +342,27 @@ func TestGroupByKeysAreChargedToTheItemBudget(t *testing.T) { ``)) } +// functionCallKey reuses one scratch buffer for the transform, and a single +// call with a large argument must not leave its buffer behind. +func TestFunctionCallKeyDropsAnOversizedBuffer(t *testing.T) { + rt := &runtime{transformState: &transformState{itemIDs: map[any]int{}}} + f := &userFunction{name: xdm.QName{Local: "f"}} + big := xdm.One(xdm.NewString(strings.Repeat("x", 4*maxKeyBufCap))) + if _, err := rt.functionCallKey(f, []xdm.Sequence{big}); err != nil { + t.Fatal(err) + } + if c := cap(rt.keyBuf); c > 64<<10 { + t.Errorf("keyBuf kept %d bytes after one large call, want at most 64 KB", c) + } + small := xdm.One(xdm.NewString("x")) + if _, err := rt.functionCallKey(f, []xdm.Sequence{small}); err != nil { + t.Fatal(err) + } + if cap(rt.keyBuf) == 0 { + t.Error("a small key's buffer should be kept for reuse") + } +} + // internBindings is called for every literal element while compiling, and a // hit must not allocate: building the key in a strings.Builder cost one // allocation per call and raised CEN's compile allocation by 12-14%. From 702c6e6a24eef3f9884562d7a950a49411109a50 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:18:13 -0400 Subject: [PATCH 51/98] Refuse deeply nested maps and arrays instead of overflowing the stack --- xdm/maparray.go | 34 ++++++++++++++++++++++----- xpath/deepnest_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++ xpath/fn_31.go | 12 ++++++++++ xpath/fn_map.go | 8 ++++++- xpath/fn_serialize.go | 36 ++++++++++++++++++++++++++++ xslt/apply.go | 8 +++++++ xslt/onempty.go | 18 +++++++------- xslt/runtime.go | 2 ++ xslt/serialize.go | 12 +--------- 9 files changed, 155 insertions(+), 28 deletions(-) create mode 100644 xpath/deepnest_test.go diff --git a/xdm/maparray.go b/xdm/maparray.go index 4ca2093a..f7fc93ff 100644 --- a/xdm/maparray.go +++ b/xdm/maparray.go @@ -434,18 +434,40 @@ func (a *ArrayItem) Members() []Sequence { return out } +// AppendMembers appends the members to dst and returns it. It is Members +// without the intermediate copy, for a caller that is building a new slice +// anyway. +func (a *ArrayItem) AppendMembers(dst []Sequence) []Sequence { + return append(dst, a.members...) +} + // Flatten replaces every array in a sequence with its members, recursively, // which is what array:flatten and the function conversion rules do. func Flatten(seq Sequence) Sequence { out := make(Sequence, 0, len(seq)) - for _, it := range seq { - arr, ok := it.(*ArrayItem) - if !ok { - out = append(out, it) + // An explicit stack rather than recursion: an array nested a million deep + // is a value a query can build, and recursing on it overflows the Go stack. + type frame struct { + seqs []Sequence // the member sequences still to walk + i int // next item within seqs[0] + } + stack := []frame{{seqs: []Sequence{seq}}} + for len(stack) > 0 { + f := &stack[len(stack)-1] + if len(f.seqs) == 0 { + stack = stack[:len(stack)-1] continue } - for _, m := range arr.members { - out = append(out, Flatten(m)...) + if f.i == len(f.seqs[0]) { + f.seqs, f.i = f.seqs[1:], 0 + continue + } + it := f.seqs[0][f.i] + f.i++ + if arr, ok := it.(*ArrayItem); ok { + stack = append(stack, frame{seqs: arr.members}) + } else { + out = append(out, it) } } return out diff --git a/xpath/deepnest_test.go b/xpath/deepnest_test.go new file mode 100644 index 00000000..6fcc5f59 --- /dev/null +++ b/xpath/deepnest_test.go @@ -0,0 +1,53 @@ +package xpath + +import ( + "errors" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// A value nested far past any real document must be refused with the +// resource-limit error, not recurse until the Go stack overflows (fatal, and +// not recoverable). The depth is just past the smallest bound, so a reverted +// fix fails on the assertion rather than crashing the test binary. +func TestDeepNestedValuesAreRefused(t *testing.T) { + const depth = 20000 + nest := func(wrap func(xdm.Sequence) xdm.Item) xdm.Sequence { + v := xdm.One(xdm.NewInteger(1)) + for i := 0; i < depth; i++ { + v = xdm.One(wrap(v)) + } + return v + } + arrays := nest(func(s xdm.Sequence) xdm.Item { return xdm.NewArray(s) }) + maps := nest(func(s xdm.Sequence) xdm.Item { + m := xdm.NewMap() + m, _ = m.Put(xdm.NewString("k"), s) + return m + }) + for _, tc := range []struct{ name, query string }{ + {"json array", `serialize($a, map{"method":"json"})`}, + {"json map", `serialize($m, map{"method":"json"})`}, + {"adaptive array", `serialize($a, map{"method":"adaptive"})`}, + {"adaptive map", `serialize($m, map{"method":"adaptive"})`}, + {"deep-equal array", `deep-equal($a, $a)`}, + {"deep-equal map", `deep-equal($m, $m)`}, + {"map:find", `Q{http://www.w3.org/2005/xpath-functions/map}find($m, "nope")`}, + } { + c, err := CompileWith(tc.query, CompileOptions{Version: XPath31}) + if err != nil { + t.Fatal(err) + } + ctx := NewContext(nil, Builtins()) + ctx = ctx.WithVar(xdm.QName{Local: "a"}, arrays).WithVar(xdm.QName{Local: "m"}, maps) + if _, err := c.Eval(ctx); err == nil || !errors.Is(err, xdm.ErrResourceLimit) { + t.Errorf("%s: got %v, want an ErrResourceLimit refusal", tc.name, err) + } + } + + // Flatten is iterative: the same value flattens to its one leaf. + if got := xdm.Flatten(arrays); len(got) != 1 { + t.Errorf("Flatten of %d nested arrays: %d items, want 1", depth, len(got)) + } +} diff --git a/xpath/fn_31.go b/xpath/fn_31.go index 2b18e2e9..1a5a5cfe 100644 --- a/xpath/fn_31.go +++ b/xpath/fn_31.go @@ -1203,6 +1203,12 @@ func deepEqualMapArray(ctx *Context, x, y xdm.Item) (eq, handled bool, err error } func deepEqualArrays(ctx *Context, a, b *xdm.ArrayItem) (bool, error) { + // Nested values recurse here; bound them like any other recursion. + if err := ctx.checkDepth(); err != nil { + return false, err + } + ctx.Depth++ + defer func() { ctx.Depth-- }() if a.Len() != b.Len() { return false, nil } @@ -1220,6 +1226,12 @@ func deepEqualArrays(ctx *Context, a, b *xdm.ArrayItem) (bool, error) { } func deepEqualMaps(ctx *Context, a, b *xdm.MapItem) (bool, error) { + // Nested values recurse here; bound them like any other recursion. + if err := ctx.checkDepth(); err != nil { + return false, err + } + ctx.Depth++ + defer func() { ctx.Depth-- }() if a.Len() != b.Len() { return false, nil } diff --git a/xpath/fn_map.go b/xpath/fn_map.go index 1c9ba972..99cef43c 100644 --- a/xpath/fn_map.go +++ b/xpath/fn_map.go @@ -182,7 +182,7 @@ func registerMapFuncs(l *Library) { // one array. Anything that is neither a map nor an array is passed over // rather than being a type error, which is what makes it usable on a // json-doc whose shape is not known in advance. - l.registerMap("find", []int{2}, func(_ *Context, args []xdm.Sequence) (xdm.Sequence, error) { + l.registerMap("find", []int{2}, func(ctx *Context, args []xdm.Sequence) (xdm.Sequence, error) { key, err := argMapKey(args, 1) if err != nil { return nil, err @@ -190,6 +190,12 @@ func registerMapFuncs(l *Library) { var found []xdm.Sequence var walk func(seq xdm.Sequence) error walk = func(seq xdm.Sequence) error { + // Recursion on the value's own nesting, so bounded like any other. + if err := ctx.checkDepth(); err != nil { + return err + } + ctx.Depth++ + defer func() { ctx.Depth-- }() for _, it := range seq { switch v := it.(type) { case *xdm.MapItem: diff --git a/xpath/fn_serialize.go b/xpath/fn_serialize.go index 0dac0e69..87a19f9a 100644 --- a/xpath/fn_serialize.go +++ b/xpath/fn_serialize.go @@ -1868,6 +1868,13 @@ func jsonFitsInline(seqs []xdm.Sequence) bool { } func writeJSONItem(sb *serializeSink, it xdm.Item, opts serializeOptions, depth int) error { + switch it.(type) { + case *xdm.ArrayItem, *xdm.MapItem: + if err := sb.enter(); err != nil { + return err + } + defer sb.leave() + } switch v := it.(type) { case *xdm.ArrayItem: members := v.Members() @@ -2281,6 +2288,13 @@ func writeAdaptiveSeq(sb *serializeSink, seq xdm.Sequence, opts serializeOptions } func writeAdaptiveItem(sb *serializeSink, it xdm.Item, opts serializeOptions) { + switch it.(type) { + case *xdm.ArrayItem, *xdm.MapItem: + if sb.enter() != nil { + return + } + defer sb.leave() + } switch v := it.(type) { case *xdm.MapItem: sb.WriteString("map{") @@ -2748,8 +2762,30 @@ type serializeSink struct { // reserve is allowance drawn from the shared counter and not yet spent. // See sinkReserveBlock. reserve int + // depth counts the maps and arrays open at this point; see enter. + depth int +} + +// enter opens one level of map or array nesting, refusing past maxJSONDepth. +// The writers recurse, so a value nested a million deep is a million Go stack +// frames and a fatal stack overflow rather than an error. A caller that gets +// nil must call leave when the level closes. +func (s *serializeSink) enter() error { + if s.depth >= maxJSONDepth { + err := fmt.Errorf("XPDY0130: serialization nests maps and arrays more "+ + "than %d levels deep: %w", maxJSONDepth, xdm.ErrResourceLimit) + if s.e == nil { + s.e = err + } + s.reserve = 0 + return err + } + s.depth++ + return nil } +func (s *serializeSink) leave() { s.depth-- } + // sinkReserveBlock is how much allowance the sink draws from the shared // counter at a time. // diff --git a/xslt/apply.go b/xslt/apply.go index 3ca74c57..5c259260 100644 --- a/xslt/apply.go +++ b/xslt/apply.go @@ -903,6 +903,14 @@ func (rt *runtime) functionCallKey(f *userFunction, args []xdm.Sequence) ([]byte // appendIdenticalKey appends the key of one sequence to b. Every atomic key is // length-prefixed, so a string holding a delimiter cannot pass for structure. func (rt *runtime) appendIdenticalKey(b []byte, s xdm.Sequence) ([]byte, error) { + // The walk recurses on the argument's own nesting. Past this depth the + // call is simply not cached, which is what any other unkeyable argument + // gets, rather than a stack overflow. + if rt.keyDepth >= 500 { + return nil, errNotCacheable + } + rt.keyDepth++ + defer func() { rt.keyDepth-- }() var err error for _, it := range s { b = append(b, 0x1e) diff --git a/xslt/onempty.go b/xslt/onempty.go index ffb094dd..728f61b7 100644 --- a/xslt/onempty.go +++ b/xslt/onempty.go @@ -119,11 +119,10 @@ func deemedEmpty(it xdm.Item) bool { case *xdm.ArrayItem: // "an array where the result of flattening the array is either an // empty sequence, or a sequence in which every item is deemed empty". - for _, m := range v.Members() { - for _, sub := range m { - if !deemedEmpty(sub) { - return false - } + // Flatten is iterative, so a deeply nested array cannot overflow the stack. + for _, sub := range xdm.Flatten(xdm.Sequence{v}) { + if !deemedEmpty(sub) { + return false } } return true @@ -160,11 +159,10 @@ func vacuous(it xdm.Item) bool { return false } case *xdm.ArrayItem: - for _, m := range v.Members() { - for _, sub := range m { - if !vacuous(sub) { - return false - } + // Flatten is iterative, so a deeply nested array cannot overflow the stack. + for _, sub := range xdm.Flatten(xdm.Sequence{v}) { + if !vacuous(sub) { + return false } } return true diff --git a/xslt/runtime.go b/xslt/runtime.go index fd882ee5..f0c01b89 100644 --- a/xslt/runtime.go +++ b/xslt/runtime.go @@ -171,6 +171,8 @@ type transformState struct { itemIDs map[any]int // keyBuf is the scratch buffer functionCallKey builds keys in. keyBuf []byte + // keyDepth is how deep appendIdenticalKey has recursed. + keyDepth int // nsPairs is the scratch list literalElemInstr.Execute collects an // element's namespace bindings in. nsPairs []string diff --git a/xslt/serialize.go b/xslt/serialize.go index e117ebdd..f1610440 100644 --- a/xslt/serialize.go +++ b/xslt/serialize.go @@ -2813,15 +2813,5 @@ func flattenArrays(seq xdm.Sequence) xdm.Sequence { if !has { return seq } - out := make(xdm.Sequence, 0, len(seq)) - for _, it := range seq { - if a, ok := it.(*xdm.ArrayItem); ok { - for _, m := range a.Members() { - out = append(out, flattenArrays(m)...) - } - continue - } - out = append(out, it) - } - return out + return xdm.Flatten(seq) } From 59ac5692b797994e29398207f3d19bba0092ffe1 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:18:19 -0400 Subject: [PATCH 52/98] array:append, insert-before and join copy members once --- xpath/arrayappend_alloc_test.go | 44 +++++++++++++++++++++++++++++++++ xpath/fn_array.go | 15 +++++------ 2 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 xpath/arrayappend_alloc_test.go diff --git a/xpath/arrayappend_alloc_test.go b/xpath/arrayappend_alloc_test.go new file mode 100644 index 00000000..b855275c --- /dev/null +++ b/xpath/arrayappend_alloc_test.go @@ -0,0 +1,44 @@ +package xpath + +import ( + "runtime" + "testing" + "unsafe" + + "github.com/knroy/go-xml/v2/xdm" +) + +// array:append, array:insert-before and array:join must copy the members once, +// not once in Members() and again into the result. +func TestArrayBuildersCopyMembersOnce(t *testing.T) { + const n, runs = 10000, 20 + members := make([]xdm.Sequence, n) + for i := range members { + members[i] = xdm.One(xdm.NewInteger(int64(i))) + } + arr := xdm.One(xdm.NewArray(members...)) + oneCopy := uint64(n * unsafe.Sizeof(xdm.Sequence(nil))) + const ns = "Q{http://www.w3.org/2005/xpath-functions/array}" + for _, q := range []string{ + ns + "append($a, 1)", ns + "insert-before($a, 5, 1)", ns + "join(($a))", + } { + c, err := CompileWith(q, CompileOptions{Version: XPath31}) + if err != nil { + t.Fatal(err) + } + ctx := NewContext(nil, Builtins()).WithVar(xdm.QName{Local: "a"}, arr) + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + for i := 0; i < runs; i++ { + if _, err := c.Eval(ctx); err != nil { + t.Fatal(err) + } + } + runtime.ReadMemStats(&after) + per := (after.TotalAlloc - before.TotalAlloc) / runs + if per > oneCopy*3/2 { + t.Errorf("%s allocated %d bytes per call; one copy of the members is %d", q, per, oneCopy) + } + } +} diff --git a/xpath/fn_array.go b/xpath/fn_array.go index 4586a541..22b76c7f 100644 --- a/xpath/fn_array.go +++ b/xpath/fn_array.go @@ -70,7 +70,8 @@ func registerArrayFuncs(l *Library) { if err != nil { return nil, err } - return xdm.One(xdm.NewArray(append(a.Members(), seqArg(args, 1))...)), nil + members := a.AppendMembers(make([]xdm.Sequence, 0, a.Len()+1)) + return xdm.One(xdm.NewArray(append(members, seqArg(args, 1))...)), nil }) // array:subarray($array, $start as xs:integer[, $length as xs:integer]) @@ -161,11 +162,11 @@ func registerArrayFuncs(l *Library) { "array:insert-before: position %d is outside the array's bounds (1 to %d)", i, a.Len()+1) } - old := a.Members() - members := make([]xdm.Sequence, 0, len(old)+1) - members = append(members, old[:i-1]...) - members = append(members, seqArg(args, 2)) - members = append(members, old[i-1:]...) + members := make([]xdm.Sequence, 0, a.Len()+1) + members = a.AppendMembers(members) + members = append(members, nil) + copy(members[i:], members[i-1:]) + members[i-1] = seqArg(args, 2) return xdm.One(xdm.NewArray(members...)), nil }) @@ -216,7 +217,7 @@ func registerArrayFuncs(l *Library) { return nil, xdm.ErrType( "array:join: expected an array, got %s", it.TypeName()) } - members = append(members, a.Members()...) + members = a.AppendMembers(members) } return xdm.One(xdm.NewArray(members...)), nil }) From c1c2bb926cf245047ac2ca8fa121a5206f57a076 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:19:56 -0400 Subject: [PATCH 53/98] Bound regex program size, charge replace before building, cap picture widths, skip caching long keys --- xpath/cache.go | 6 +++ xpath/collation.go | 4 +- xpath/fn_misc.go | 8 ++++ xpath/fn_regex.go | 19 ++++++++-- xpath/regex_backref.go | 2 +- xpath/regex_backtrack.go | 5 ++- xpath/regex_bound_test.go | 80 +++++++++++++++++++++++++++++++++++++++ xpath/regex_size.go | 60 +++++++++++++++++++++++++++++ xsd/pattern.go | 3 +- xsd/pattern_bound_test.go | 21 ++++++++++ 10 files changed, 200 insertions(+), 8 deletions(-) create mode 100644 xpath/regex_bound_test.go create mode 100644 xpath/regex_size.go create mode 100644 xsd/pattern_bound_test.go diff --git a/xpath/cache.go b/xpath/cache.go index 588a40f3..639b43aa 100644 --- a/xpath/cache.go +++ b/xpath/cache.go @@ -31,6 +31,12 @@ import "sync" // contended case that would justify an RWMutex is a long read, which this is // not. Mutex is also the smaller and simpler primitive, which matters for a // type instantiated three times over three different value shapes. +// maxCachedKey is the longest key worth caching. The caches bound entries, not +// bytes, and the key (a pattern, a collation URI) can come from document data: +// 1024 entries of long keys, each with a compiled program behind it, retained +// gigabytes. A key this long is simply recompiled on each use. +const maxCachedKey = 1024 + type boundedCache[K comparable, V any] struct { mu sync.Mutex items map[K]V diff --git a/xpath/collation.go b/xpath/collation.go index 1fcaea97..a2f4e8b4 100644 --- a/xpath/collation.go +++ b/xpath/collation.go @@ -820,7 +820,9 @@ func ucaCollationFor(uri string) (Collation, error) { return e.c, e.err } c, err := parseUCACollation(uri) - ucaCache.Put(uri, ucaCacheEntry{c, err}) + if len(uri) <= maxCachedKey { + ucaCache.Put(uri, ucaCacheEntry{c, err}) + } return c, err } diff --git a/xpath/fn_misc.go b/xpath/fn_misc.go index c7ba5b70..1fecaa0c 100644 --- a/xpath/fn_misc.go +++ b/xpath/fn_misc.go @@ -1313,6 +1313,10 @@ func defaultPresentation(comp byte) string { return "1" } +// maxPictureWidth bounds a width modifier's number. The padding it asks for is +// built before any budget sees it: [D01,1000000000] allocated 2.8 GB. +const maxPictureWidth = 1 << 16 + // checkWidthModifier rejects a width modifier whose syntax is wrong, which is // XTDE1340. Both bounds must be "*" or an integer greater than zero. func checkWidthModifier(width string) error { @@ -1333,6 +1337,10 @@ func checkWidthModifier(width string) error { return fmt.Errorf("FOFD1340: malformed width modifier %q", width) } n = n*10 + int(c-'0') + if n > maxPictureWidth { + return fmt.Errorf("FOFD1340: width modifier %q asks for more than %d "+ + "characters: %w", width, maxPictureWidth, xdm.ErrResourceLimit) + } } if n == 0 { return fmt.Errorf("FOFD1340: width modifier %q must be greater than zero", width) diff --git a/xpath/fn_regex.go b/xpath/fn_regex.go index a231d5c7..f49d5428 100644 --- a/xpath/fn_regex.go +++ b/xpath/fn_regex.go @@ -133,8 +133,18 @@ func registerRegexFuncs(l *Library) { if err != nil { return nil, err } + // The result is charged before it is built: 500 KB of input with a 4 KB + // replacement allocated 12.8 GB before the old after-the-fact charge + // refused it. The estimate is the input plus one replacement per match; + // only what the built string exceeds it by is charged afterwards. + n := 0 + re.ReplaceAllStringFunc(s, func(string) string { n++; return "" }) + est := len(s) + n*len(goRepl) + if err := ctx.countBytes(est); err != nil { + return nil, err + } out := re.ReplaceAllString(s, goRepl) - if err := ctx.countBytes(len(out)); err != nil { + if err := ctx.countBytes(len(out) - est); err != nil { return nil, err } return strSeq(out), nil @@ -299,6 +309,9 @@ func compileXPathRegexp(pattern, flags string, v Version) (*regexp.Regexp, error // the check and the insert under one lock: doing them separately let concurrent // callers interleave between the two and carry the table past its limit. func storeRegex(key string, v any) { + if len(key) > maxCachedKey { + return + } regexCache.Put(key, v) } @@ -391,7 +404,7 @@ func buildRegexp(pattern, flags string, v Version) (*regexp.Regexp, error) { // XPath's fn:matches is a *containment* test, unlike XML Schema's pattern // facet which is anchored. Go's MatchString is also containment, so no // anchoring is added here. - re, err := regexp.Compile(translated) + re, err := CompileBounded(translated) if err != nil { // A repeat count RE2 will not accept is still a valid XML Schema // pattern — "a{2147483647}" simply matches nothing any input can @@ -399,7 +412,7 @@ func buildRegexp(pattern, flags string, v Version) (*regexp.Regexp, error) { // when it was merely enormous, so the count is clamped to one past // RE2's limit, which matches exactly what the original would. if clamped, ok := clampRepeatCounts(translated); ok { - if re2, err2 := regexp.Compile(clamped); err2 == nil { + if re2, err2 := CompileBounded(clamped); err2 == nil { return re2, nil } } diff --git a/xpath/regex_backref.go b/xpath/regex_backref.go index d497cc08..a9c92dde 100644 --- a/xpath/regex_backref.go +++ b/xpath/regex_backref.go @@ -653,7 +653,7 @@ func buildBackrefRegexp(pattern, flags string, v Version) (*backrefRegexp, error if len(goFlags) > 0 { translated = "(?" + strings.Join(goFlags, "") + ")" + translated } - re, err := regexp.Compile(translated) + re, err := CompileBounded(translated) if err != nil { return nil, fmt.Errorf( "FORX0002: invalid regular expression %q: %w", original, err) diff --git a/xpath/regex_backtrack.go b/xpath/regex_backtrack.go index 0973c81d..d078d2a4 100644 --- a/xpath/regex_backtrack.go +++ b/xpath/regex_backtrack.go @@ -616,6 +616,9 @@ func (p *btParser) compileCharAtom(src string) (btNode, error) { // storeCharAtom adds an entry, clearing the cache first if it is full. func storeCharAtom(key charAtomKey, v any) { + if len(key.src) > maxCachedKey { + return + } charAtomCache.Put(key, v) } @@ -633,7 +636,7 @@ func buildCharAtom(src string, dotAll, fold bool) (*regexp.Regexp, error) { // affect a dot the pattern escaped, which is a literal either way. What it // does buy is that "\n" is matched by a negated class rather than falling // foul of RE2's line handling. - re, err := regexp.Compile(prefix + `\A(?:` + translated + `)\z`) + re, err := CompileBounded(prefix + `\A(?:` + translated + `)\z`) if err != nil { return nil, fmt.Errorf("FORX0002: invalid regular expression %q: %w", src, err) } diff --git a/xpath/regex_bound_test.go b/xpath/regex_bound_test.go new file mode 100644 index 00000000..192e644d --- /dev/null +++ b/xpath/regex_bound_test.go @@ -0,0 +1,80 @@ +package xpath + +import ( + "errors" + "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// Sequential \pL{1000} passes Go's nested-repeat check, so a 904-byte pattern +// compiled to a 5 MB program and an 18 KB one needed 462 MB to compile. +func TestRegexProgramSizeIsBounded(t *testing.T) { + big := strings.Repeat(`\pL{1000}`, 150) + _, err := CompileRegexp(big, "") + if !errors.Is(err, xdm.ErrResourceLimit) || !strings.Contains(err.Error(), "FORX0002") { + t.Fatalf("150 x \\pL{1000}: want FORX0002 + ErrResourceLimit, got %v", err) + } + if _, err := CompileBounded(big); !errors.Is(err, xdm.ErrResourceLimit) { + t.Fatalf("CompileBounded: want ErrResourceLimit, got %v", err) + } + if _, err := CompileRegexp(strings.Repeat(`\pL{1000}`, 50), ""); err != nil { + t.Fatalf("50 x \\pL{1000} is under the bound and must compile: %v", err) + } +} + +// replace() was charged after it had built the result: 500 KB of input with a +// 4 KB replacement allocated 12.8 GB before the budget refused it. +func TestReplaceChargesBeforeBuilding(t *testing.T) { + ctx := NewContext(nil, Builtins()) + var m0, m1 runtime.MemStats + runtime.ReadMemStats(&m0) + _, err := Eval(`replace(string-join(for $i in 1 to 300000 return 'a', ''), '.', `+ + `string-join(for $i in 1 to 4096 return 'b', ''))`, ctx, nil) + runtime.ReadMemStats(&m1) + if !errors.Is(err, xdm.ErrResourceLimit) { + t.Fatalf("want ErrResourceLimit, got %v", err) + } + if mb := (m1.TotalAlloc - m0.TotalAlloc) >> 20; mb > 2000 { + t.Errorf("replace allocated %d MB before refusing", mb) + } +} + +// A width modifier's minimum drove strings.Repeat with nothing charging it: +// [D01,1000000000] allocated 2.8 GB. +func TestPictureWidthIsBounded(t *testing.T) { + ctx := NewContext(nil, Builtins()) + ctx = ctx.WithEnv(func(e *Env) { e.LibraryVersion = XPath31 }) + _, err := Eval(`format-date(xs:date('2000-01-01'), '[D01,100000000]')`, ctx, nil) + if !errors.Is(err, xdm.ErrResourceLimit) || !strings.Contains(err.Error(), "FOFD1340") { + t.Fatalf("want FOFD1340 + ErrResourceLimit, got %v", err) + } + if got := mustEval(t, ctx, `format-date(xs:date('2000-01-01'), '[D01,5]')`); got != "00001" { + t.Errorf("ordinary width changed: %q", got) + } +} + +// The caches bound entries, not bytes, so a long key is not cached at all. +func TestLongKeysAreNotCached(t *testing.T) { + long := strings.Repeat("a", 1100) + n := regexCache.Len() + if _, err := compileXPathRegexp(long, "", XPath20); err != nil { + t.Fatal(err) + } + if regexCache.Len() != n { + t.Errorf("regexCache cached a %d-byte pattern", len(long)) + } + m := ucaCache.Len() + _, _ = ucaCollationFor("http://www.w3.org/2013/collation/UCA?" + long) + if ucaCache.Len() != m { + t.Errorf("ucaCache cached a failed %d-byte URI", len(long)) + } + k := charAtomCache.Len() + p := &btParser{} + _, _ = p.compileCharAtom(strings.Repeat("[a]", 400)) + if charAtomCache.Len() != k { + t.Errorf("charAtomCache cached a long atom") + } +} diff --git a/xpath/regex_size.go b/xpath/regex_size.go new file mode 100644 index 00000000..9dce43a2 --- /dev/null +++ b/xpath/regex_size.go @@ -0,0 +1,60 @@ +package xpath + +import ( + "fmt" + "regexp" + "regexp/syntax" + + "github.com/knroy/go-xml/v2/xdm" +) + +// MaxRegexpProgram bounds the compiled size of a pattern, in RE2 instructions. +// +// Go refuses a nested repeat product over 1000 but not a sequence of them, so +// \pL{1000} written a hundred times (904 bytes) compiled to a program that +// retained 5 MB, and an 18 KB pattern needed 462 MB to compile. The cap is +// checked on the parsed tree before regexp.Compile builds anything. +const MaxRegexpProgram = 100000 + +// CompileBounded is regexp.Compile that refuses a pattern whose program would +// exceed MaxRegexpProgram instructions. The refusal wraps xdm.ErrResourceLimit. +// It is shared by the XPath and XSD pattern compilers. +func CompileBounded(expr string) (*regexp.Regexp, error) { + re, err := syntax.Parse(expr, syntax.Perl) + if err != nil { + // Let regexp.Compile report it, so the message is the usual one. + return regexp.Compile(expr) + } + if n := progSize(re.Simplify(), map[*syntax.Regexp]int{}); n > MaxRegexpProgram { + return nil, fmt.Errorf("regular expression compiles to more than %d instructions: %w", + MaxRegexpProgram, xdm.ErrResourceLimit) + } + return regexp.Compile(expr) +} + +// progSize estimates the instruction count of a simplified tree. Simplify +// shares the operand of x{n}, so results are memoised by node to keep the walk +// linear in the pattern rather than in the expansion. Sums are clamped so a +// hostile pattern cannot overflow them. +func progSize(re *syntax.Regexp, memo map[*syntax.Regexp]int) int { + if n, ok := memo[re]; ok { + return n + } + n := 1 + switch re.Op { + case syntax.OpLiteral: + n = len(re.Rune) + case syntax.OpConcat, syntax.OpAlternate: + for _, s := range re.Sub { + n += progSize(s, memo) + if n > MaxRegexpProgram*2 { + n = MaxRegexpProgram * 2 + break + } + } + case syntax.OpCapture, syntax.OpStar, syntax.OpPlus, syntax.OpQuest: + n += 1 + progSize(re.Sub[0], memo) + } + memo[re] = n + return n +} diff --git a/xsd/pattern.go b/xsd/pattern.go index aff91490..0f8b6a35 100644 --- a/xsd/pattern.go +++ b/xsd/pattern.go @@ -2,7 +2,6 @@ package xsd import ( "fmt" - "regexp" "github.com/knroy/go-xml/v2/xpath" ) @@ -40,7 +39,7 @@ func compilePatternVersion(src string, version Version) (*Pattern, error) { // would reintroduce exactly the widening this wrapping exists to // prevent. The non-capturing group keeps a top-level alternation — // "a|b" — from binding more loosely than the anchors. - re, err := regexp.Compile(`\A(?:` + translated + `)\z`) + re, err := xpath.CompileBounded(`\A(?:` + translated + `)\z`) if err != nil { return nil, fmt.Errorf("pattern %q: %w", src, err) } diff --git a/xsd/pattern_bound_test.go b/xsd/pattern_bound_test.go new file mode 100644 index 00000000..9af1b14e --- /dev/null +++ b/xsd/pattern_bound_test.go @@ -0,0 +1,21 @@ +package xsd + +import ( + "errors" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// A pattern facet had no bound at all: 16 types holding sequential \p{L}{1000} +// repeats in a 354 KB schema retained 1.4 GB. +func TestPatternProgramSizeIsBounded(t *testing.T) { + _, err := compilePattern(strings.Repeat(`\p{L}{1000}`, 150)) + if !errors.Is(err, xdm.ErrResourceLimit) || !strings.Contains(err.Error(), "instructions") { + t.Fatalf("want a program-size refusal wrapping ErrResourceLimit, got %v", err) + } + if _, err := compilePattern(strings.Repeat(`\p{L}{1000}`, 50)); err != nil { + t.Fatalf("a pattern under the bound must compile: %v", err) + } +} From 9c3a9839b568cf9434a7b801a04ba59036ffc62f Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:23:51 -0400 Subject: [PATCH 54/98] xdm: return the input from ProcessXInclude only when it is a frozen parse A built tree can still be changed, and the caller is promised a tree separate from it. --- xdm/xinclude.go | 4 +++- xdm/xinclude_nocopy_test.go | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/xdm/xinclude.go b/xdm/xinclude.go index 40132552..78aa0169 100644 --- a/xdm/xinclude.go +++ b/xdm/xinclude.go @@ -113,8 +113,10 @@ func ProcessXInclude(tree *Tree, opts XIncludeOptions) (*Tree, error) { if base == "" { base = tree.Root.DocumentURI() } - if !mentionsXInclude(tree) { + if tree.frozen && !mentionsXInclude(tree) { // Nothing to replace: the copy processTree would make is the tree. + // Only a parsed tree is returned as is; a built one can still be + // changed, and the caller is promised a tree separate from it. return tree, nil } p := &includeProc{opts: opts, budget: &entityBudget{}} diff --git a/xdm/xinclude_nocopy_test.go b/xdm/xinclude_nocopy_test.go index dd25c3d6..07b12134 100644 --- a/xdm/xinclude_nocopy_test.go +++ b/xdm/xinclude_nocopy_test.go @@ -15,6 +15,12 @@ func TestXIncludeSkipsCopyWithoutXIElements(t *testing.T) { if err != nil || out != tree { t.Fatalf("plain tree: got %p (tree %p), err %v; want the tree itself", out, tree, err) } + // A tree that is not a frozen parse can still be changed, so it gets its + // own copy even with nothing to include. + built := &Tree{Root: Copy(tree.Root)} + if out, err := ProcessXInclude(built, XIncludeOptions{}); err != nil || out == built { + t.Fatalf("built tree: got %p (tree %p), err %v; want a new tree", out, built, err) + } big := "" + strings.Repeat("text", 20000) + "" parseBytes := allocBytes(func() { parseWithBase(t, "mem:/big.xml", big) }) From 4611566c19615aea448e0d7fcb7a983fd8aba7d3 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sat, 31 Oct 2026 16:25:39 -0400 Subject: [PATCH 55/98] docs: record the second memory audit's fixes Changelog rows, the security pass entries, the regex repeat claim the audit disproved, the cache and XInclude behaviour, the replace() gap, and regenerated test counts. --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ README.md | 6 +++--- docs/conformance-gaps.md | 2 +- docs/known-gaps.md | 11 +++++++++++ docs/migrating-to-v2.md | 3 ++- docs/options.md | 3 ++- docs/security.md | 16 ++++++++++++++-- docs/stats.md | 2 +- docs/testing.md | 2 +- docs/todo.md | 2 +- 10 files changed, 66 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b52bdd9..2794a46a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ Three new exported functions make this a minor release. |---|---|---| | `xpath.SerializeJSONTo`, `xpath.SerializeAdaptiveTo` | The json and adaptive methods written to an `io.Writer` instead of returned as a string. | [`599b2f9`][599b2f9] | | `xdm.Node.AddNamespaces` | Sets several namespace bindings on an element in one call. | [`b42f04a`][b42f04a] | +| `xdm.ArrayItem.AppendMembers` | Appends an array's members to a slice without an intermediate copy. | [`59ac569`][59ac569] | +| `xpath.CompileBounded`, `xpath.MaxRegexpProgram` | `regexp.Compile` that refuses a program over 100,000 instructions; used by XPath and XSD patterns. | [`c1c2bb9`][c1c2bb9] | +| `xpath.Context.NodesRefused`, `ItemsRefused` | Whether the node budget was crossed, and whether a running item total is past the item bound, without charging either. | [`4e8f0a5`][4e8f0a5], [`21995a6`][21995a6] | ### Changed @@ -23,6 +26,8 @@ Three new exported functions make this a minor release. |---|---|---| | `xsd` and `dtd` error messages clip document-supplied strings to 1 KB plus "..." | Nothing, unless you match messages longer than 1 KB. Message wording is not part of the contract. | [`20958aa`][20958aa], [`bd865e4`][bd865e4] | | json and adaptive serialization can leave partial output when an error is raised mid-document | Treat the writer's content as invalid on error, as the XML methods already required. | [`599b2f9`][599b2f9] | +| New refusals: maps and arrays nested past 1,000 levels in json/adaptive output (XPDY0130) or 500 in `deep-equal` and `map:find` (XPDY0001); regexes over 100,000 instructions (FORX0002, or a pattern error in XSD); picture widths over 65,536 (FOFD1340); `xsl:key` and `group-by` keys past 5,000,000 retained (XPDY0130) | Nothing for real inputs: the largest W3C suite regex is 4,313 instructions. All wrap `xdm.ErrResourceLimit`. | [`702c6e6`][702c6e6], [`c1c2bb9`][c1c2bb9], [`21995a6`][21995a6] | +| `xdm.ProcessXInclude` returns a parsed tree unchanged when it has no XInclude element | Nothing; a parsed tree is frozen. A built tree is still copied. | [`e139525`][e139525], [`9c3a983`][9c3a983] | ### Fixed @@ -32,6 +37,11 @@ Three new exported functions make this a minor release. | The DTD validator built every element's path, quadratic in depth | A valid 990-deep chain of 4 KB names allocated 2 GB. The path is rendered only for a failure, text unchanged. | [`8dcb07f`][8dcb07f] | | DTD validity messages kept `#FIXED`, enum and ID values whole | 200 mismatches on a 1 MB `#FIXED` value kept 105 MB. `fail` clips to 1 KB. | [`bd865e4`][bd865e4] | | The nested `fn:transform` cache kept compiles for dead resolvers | A fresh resolver per call kept 49 MB on DocBook. Resolvers are weak keys and dead entries are dropped: 0.3 MB. | [`a5b385d`][a5b385d] | +| Deeply nested maps and arrays crashed the process | `serialize(..., json)` or `deep-equal` on a 400,000-deep array from a 96-byte query overflowed the Go stack. They are refused with a resource-limit error; `xdm.Flatten` is iterative. Also in v2.0.3. | [`702c6e6`][702c6e6] | +| A built-in `shallow-copy` rule returned a truncated result past `MaxNodes` with no error | A 1M-element identity copy emptied a third of its elements and exited 0. The transform now fails with XPDY0130 whenever the node budget was crossed. Also in v2.0.3. | [`4e8f0a5`][4e8f0a5] | +| Sequential regex repeats compiled to huge programs | `\pL{1000}` repeated 150 times passes Go's 1,000-repeat check: 181 KB of patterns kept 984 MB. Programs over 100,000 instructions are refused in XPath and XSD. | [`c1c2bb9`][c1c2bb9] | +| `fn:replace`, picture widths, `xsl:key` and `group-by` grew past their budgets | `replace` charged after building (12.8 GB for 500 KB in), `[D01,1000000000]` allocated 2.8 GB, and `use="1 to 1000"` over 100k nodes 3.2 GB. Each is charged or refused first. | [`c1c2bb9`][c1c2bb9], [`21995a6`][21995a6] | +| Caches bounded by entries held gigabytes of long keys or documents | Regex, collation and char-atom keys over 1 KB are not cached; the `FileResolver` document cache is also bounded by 32 MB of source. | [`c1c2bb9`][c1c2bb9], [`de4a515`][de4a515] | ### Changed — memory @@ -52,6 +62,11 @@ Three new exported functions make this a minor release. | CDATA sections, character maps and `fn:serialize` escaping copied per node | Written straight to the sink: xmark CDATA 20.4 MB → 34 KB, character map 16.5 MB → 34 KB. | [`cf65d62`][cf65d62], [`599b2f9`][599b2f9] | | c14n `Bytes` and `Result.String` doubled a buffer from empty; `Serialize` allocated 32 KB a call | Output chunks are joined once and the write buffer is pooled: c14n 3.7× → 2.1× the output, a small result 33.6 → 0.7 KB. | [`f235795`][f235795], [`9496810`][9496810], [`37206b1`][37206b1], [`c501fc4`][c501fc4], [`1f18b1b`][1f18b1b] | | The `xslt` benchmarks read inputs no checkout has, so they always skipped | They run XRechnung and Peppol CEN from `testdata/bench`. | [`beb6072`][beb6072] | +| `-xinclude` copied every tree, and included documents twice, even with nothing to include | A parsed tree with no XInclude element is returned as is and inclusions are built into the destination: `parse-100mb` 514 → 0 MB. | [`e139525`][e139525] | +| RELAX NG copied its name memo on every store | Stores land in a pending map published as a snapshot: 1,024 stores 47.5 → under 2 MB. | [`9f0db9c`][9f0db9c] | +| DTD validation built a state-set map, a name slice and an attribute map per element | Small sets dedup by scan, the slice is reused, no ATTLIST skips the map: 12,062 → 8,062 allocations for 6,001 elements. | [`5dc120c`][5dc120c] | +| `xsdnet.HTTPResolver` built a new transport per fetch | One guarded transport is shared, so connections are reused; the SSRF guard is unchanged. | [`da05b16`][da05b16] | +| Typed copies pinned dropped schemas; `array:append` copied twice; line index, interning key and memo key kept slack | Weak typing-name slots, one copy, clipped index, reused key buffer, oversized memo buffer dropped. | [`967dfb3`][967dfb3], [`59ac569`][59ac569], [`3b71501`][3b71501], [`69414ae`][69414ae], [`887d5d3`][887d5d3] | ## v2.0.3 — 2026-10-30 @@ -1712,3 +1727,18 @@ here so every entry in this file sits under a release. [c501fc4]: https://github.com/knroy/go-xml/commit/c501fc4 [1f18b1b]: https://github.com/knroy/go-xml/commit/1f18b1b [beb6072]: https://github.com/knroy/go-xml/commit/beb6072 +[59ac569]: https://github.com/knroy/go-xml/commit/59ac569 +[c1c2bb9]: https://github.com/knroy/go-xml/commit/c1c2bb9 +[4e8f0a5]: https://github.com/knroy/go-xml/commit/4e8f0a5 +[21995a6]: https://github.com/knroy/go-xml/commit/21995a6 +[702c6e6]: https://github.com/knroy/go-xml/commit/702c6e6 +[e139525]: https://github.com/knroy/go-xml/commit/e139525 +[9c3a983]: https://github.com/knroy/go-xml/commit/9c3a983 +[de4a515]: https://github.com/knroy/go-xml/commit/de4a515 +[9f0db9c]: https://github.com/knroy/go-xml/commit/9f0db9c +[5dc120c]: https://github.com/knroy/go-xml/commit/5dc120c +[da05b16]: https://github.com/knroy/go-xml/commit/da05b16 +[967dfb3]: https://github.com/knroy/go-xml/commit/967dfb3 +[3b71501]: https://github.com/knroy/go-xml/commit/3b71501 +[69414ae]: https://github.com/knroy/go-xml/commit/69414ae +[887d5d3]: https://github.com/knroy/go-xml/commit/887d5d3 diff --git a/README.md b/README.md index 8eefcdae..a0e5e4ac 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ and maintains it as a project of his own. | **XSD 1.1** | 99.98% instance (26,217 of 26,222); **99.97%** schema-validity (15,350 of 15,354); opt-in via `Version11` | | **RELAX NG** | 100.00% of James Clark's spectest (965 of 965 assertions); XML and compact syntax, from Go and from `go-xml validate -rng` | | **DTD** | content models, attribute defaults, enumerations, `ID`/`IDREF`; external subset, parameter entities across both subsets, conditional sections — via `dtd.Load` with a caller-supplied resolver, nothing fetched by default | -| **Tests** | 2,887 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | +| **Tests** | 2,907 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | | **Production schemas** | UBL 2.1, UN/CEFACT CII, Factur-X/ZUGFeRD, Peppol BIS 3.0 — 88 schemas load, instances validate clean | | **API** | 1.2; the exported surface is stable and additive over 1.1, and a breaking change means 2.0 with a new module path | @@ -669,7 +669,7 @@ exact signal. `FileResolver`'s document cache exists because `fn:doc` is defined to return the *same* node for the same URI within one execution, so `doc('x') is doc('x')` -requires it. It is mutex-guarded, and now capped at 256 documents. It had been +requires it. It is mutex-guarded, and now capped at 256 documents and 32 MB of source. It had been unbounded on the reasoning that the directories the caller opened with `-allow-dir` bound it in practice — true for the size of the *set*, but the process holds every document it has ever parsed for its whole lifetime, and a @@ -1388,7 +1388,7 @@ back, is in [docs/testing.md](docs/testing.md). | method | what it catches | what it misses | |---|---|---| -| **Unit tests** (2,887 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** (2,907 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | | **Spec inventories** | features absent entirely | features present but behaving wrongly | | **Saxon differential** | subtle behavioural divergence on real stylesheets | constructs the corpora do not use | | **W3C QT3 suite** | systematic conformance across 22,054 XPath and 30,517 XQuery cases | XSLT (it is an XPath suite) | diff --git a/docs/conformance-gaps.md b/docs/conformance-gaps.md index 147c0539..2bde3419 100644 --- a/docs/conformance-gaps.md +++ b/docs/conformance-gaps.md @@ -49,7 +49,7 @@ W3C disagreements: 0 + 0 + 0 + 1 + 8 + 23 + 30 + 31 + 0 = 93. Measured 2026-09-1 -The unit-test suite is 2,887 tests. +The unit-test suite is 2,907 tests. The last two rows are not W3C suites but real-world corpora — DocBook xslTNG's diff --git a/docs/known-gaps.md b/docs/known-gaps.md index 680b6a0a..551f0099 100644 --- a/docs/known-gaps.md +++ b/docs/known-gaps.md @@ -105,6 +105,17 @@ tables being copied per scope and target nodes being walked again. Found by an external audit of v2.0.0; the figures are the `BenchmarkIdentity*` benchmarks in `xsd/`. +### `fn:replace` charges a backtracking or backreference result after building it (XPath, resource limits) + +The RE2 path of `fn:replace` charges `len(input) + matches × len(replacement)` +before it builds the result, so a hostile replacement is refused without being +allocated. The two other engines, the backreference matcher and the opt-in +backtracking one (`-backtracking-regex`), still build the result and charge it +afterwards. Their step budgets bound the number of matches, but not the size +of each replacement, so a long replacement string can allocate past +`MaxBytes` before the refusal. Found by the second memory audit +(2026-10-31), which measured only the RE2 path. + ### §19.8 streamability analysis is partially implemented (XSLT 3.0) **3 of the 23 XSLT 3.0 failures** — `si-fork-902`, `si-fork-952` and diff --git a/docs/migrating-to-v2.md b/docs/migrating-to-v2.md index 363d763b..a741babb 100644 --- a/docs/migrating-to-v2.md +++ b/docs/migrating-to-v2.md @@ -231,7 +231,8 @@ Namespace nodes exist once per element and prefix: the namespace axis, and agree for them. `xdm.ProcessXInclude` returns the included document as a new tree instead of -editing the one it is given: `tree, err = xdm.ProcessXInclude(tree, opts)`. +editing the one it is given (a parsed tree with nothing to include comes back +as it is): `tree, err = xdm.ProcessXInclude(tree, opts)`. ## xdmbuild setters removed diff --git a/docs/options.md b/docs/options.md index 907d93e7..cff27fb9 100644 --- a/docs/options.md +++ b/docs/options.md @@ -871,7 +871,8 @@ tree, err = xdm.ProcessXInclude(tree, xdm.XIncludeOptions{ `ProcessXInclude` returns the included document as a new tree, built top-down as every tree is; the tree it is given is left as it was, and node -identities do not carry over. +identities do not carry over. A parsed tree with no XInclude element has +nothing to replace and is returned as it is. **The resolver is the whole of the confinement.** `xdm` has no filesystem and no network; it can read only what a resolver hands it. `xslt.FileResolver` diff --git a/docs/security.md b/docs/security.md index 3bfbb205..b8c9ec87 100644 --- a/docs/security.md +++ b/docs/security.md @@ -610,6 +610,8 @@ what actually drives them: the resolver. `resolverCacheMax = 256` **cleared on full** rather than evicting, so past 257 distinct URIs it stopped being a bound and became an amplifier; it now evicts one entry at a time, which is also fixed below. + Since the memory audit it is also bounded by `resolverCacheMaxBytes`, 32 MB + of source (about 5x that in trees); a `Preload`ed tree is charged 0. Two further sites — `fn:transform`'s `nestedParseOptions` (`xslt/fntransform.go:531`) and `relaxng/resolve.go:141` — have the same shape @@ -1252,7 +1254,12 @@ itself is rejected without it. See Go's `regexp` is RE2. `matches('aaaa…!', '^(a+)+$')` is flat at 2–6 µs from n=24 to n=40. Go also *rejects* repeat counts over 1000, so `{1,1000000}` cannot be used to force an expansion; the XSD pattern translator allocates nothing on -nested quantifiers or 200-deep groups. +nested quantifiers or 200-deep groups. That check bounds one repeat and nested +products, **not sequential repeats**: `\pL{1000}` written 150 times compiled to +a program of millions of instructions, and 181 KB of such patterns retained +984 MB. Every pattern now goes through `xpath.CompileBounded`, which counts the +program before building it and refuses past `MaxRegexpProgram` (100,000; the +largest in the W3C suites is 4,313), in XPath and in XSD pattern facets. **Backreferences do not change this, by default.** XPath 2.0 has them and RE2 does not, and the usual way to bridge that is a backtracking engine — which is @@ -1721,7 +1728,8 @@ and facets still apply to the substituted type. static context an environment of its own; the four reads are marked in the file. - The `xpath` regex cache is bounded at 1024, as is the backtracking engine's - single-character-atom cache; the UCA collation cache is bounded at 256. All + single-character-atom cache; the UCA collation cache is bounded at 256. The + bound is on entries, so a key over 1 KB (`maxCachedKey`) is never cached. All three hold their bound under concurrent use, not merely on a single goroutine: each is a `boundedCache` (`xpath/cache.go`) that performs the full-check and the insert under one lock hold. An earlier form checked an atomic size counter @@ -1861,6 +1869,10 @@ reject* refused a legal one, and *cost* produced the right answer too slowly. - **`xsd` error messages kept document strings whole** — cost, driven by document data. `fail` and `attrName` quoted names and values in full and the validator keeps up to 100 errors, so 200 foreign attributes in a 1 MB namespace retained 105 MB, and one 15 MB value under a failing rule set retained 1.57 GB (audit 20). Strings and the formatted message are clipped to 1 KB plus "...", and the pattern and list-item errors quote through `truncate`: 0.13 MB. Pinned by `TestValidationMessagesClipDocumentStrings` in `xsd/message_clip_test.go`. - **The DTD validator built every element's path, quadratic in depth** — cost, driven by document data. The path string was rebuilt at each element whether or not anything failed: a *valid* 990-deep chain of 4 KB names allocated 2 GB. It is rendered from the ancestors only when a failure is recorded, and an IDREF keeps its element rather than a path. Its `#FIXED`, enum and ID values are clipped to 1 KB too. Pinned by `TestPathBuiltOnlyForFailures` and `TestValidityMessagesClipLongValues` in `dtd/`. +- **Deeply nested maps and arrays overflowed the Go stack** — availability, driven by a query or stylesheet. `serialize(..., map{"method":"json"})`, the adaptive method, `fn:deep-equal` and `map:find` recursed once per level, so `fold-left(1 to 400000, [], function($a,$i){[$a]})` (96 bytes) ended the process with `fatal error: stack overflow`, which no `recover` can catch; v2.0.3 has it too. Serialization refuses past 1,000 levels (the `parse-json` limit) with XPDY0130, `deep-equal` and `map:find` past the 500-level recursion bound with XPDY0001, and `xdm.Flatten` is iterative. Pinned by `TestDeepNestedValuesAreRefused` in `xpath/deepnest_test.go`. +- **A built-in rule returned a truncated result tree as a success** — correctness, driven by document size. The result builder latches a `MaxNodes` refusal, but only instruction bodies checked it, so `` over 1M elements emptied a third of them and exited 0; v2.0.3 has it too. The transform now fails with XPDY0130 whenever the node budget was crossed anywhere, through `xpath.Context.NodesRefused`. Pinned by `TestBuiltInRuleResultTreeOverrunIsRefused`. +- **`fn:replace`, picture widths, `xsl:key` and `group-by` grew past their budgets** — cost, driven by document or stylesheet data. `replace` charged its result after building it (12.8 GB for a 500 KB input and a 4 KB replacement); a `format-date` width of `[D01,1000000000]` (52 bytes) allocated 2.8 GB through `strings.Repeat`; `use="1 to 1000"` over 100k nodes built 100M key entries at 3.2 GB peak, since the item counter resets per expression. `replace` charges first on the RE2 path (the other two engines are a known gap), widths over 65,536 are FOFD1340, and keys are totalled per index and grouping against `MaxItems`. +- **Caches bounded by entry count held gigabytes** — cost, driven by document data or the host. 1,024 cached regexes of 200 KB, 256 failed 1 MB collation URIs, or 256 large `FileResolver` documents each held hundreds of MB to GB. Keys over 1 KB are not cached, and the document cache is also bounded by 32 MB of source. - **The nested `fn:transform` cache kept compiles for collected resolvers** — cost, driven by the host. The key held each resolver by identity, so a service that made one per request kept up to 16 dead compiles a stylesheet: 49 MB on DocBook after 20 resolvers. Resolvers are weak keys and `put` drops entries whose resolver is gone: 0.3 MB. Pinned by `TestTransformCache/dead_resolver_dropped`. **Seventeenth pass — normalize-space built a list of every token, 2026-10-30.** diff --git a/docs/stats.md b/docs/stats.md index cfe2db31..13b80337 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -23,7 +23,7 @@ claim is only as good as the command behind it. | figure | count | counted by | |---|---:|---| -| Unit tests | 2,887 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | +| Unit tests | 2,907 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Fuzz targets | 12 | `grep -rn '^func Fuzz' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Limit boundary tests | 14 | `grep -hc '^func Test' ./*/limits_boundary*_test.go \| awk '{n += $1} END {print n + 0}'` | diff --git a/docs/testing.md b/docs/testing.md index 88c1c0c4..28ffbab7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,7 +21,7 @@ let something through, and the column that matters is the last one. | layer | count | catches | misses | |---|---:|---|---| -| **Unit tests** | 2,887 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** | 2,907 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | | **Limit boundary tests** | 14 tests | an off-by-one or an overflow at the edge of a configurable limit | a limit nobody added to the inventory | | **Race detector** | same tests | shared state a single-goroutine run never reveals | a data race on a path no test walks | | **W3C conformance suites** | 152,241 cases | systematic divergence from the specification | what the suites do not ask about — see below | diff --git a/docs/todo.md b/docs/todo.md index 9b41a69c..62c44c6b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -23,7 +23,7 @@ Current position: | XSLT 3.0 | 99.80% — 11,495 of 11,518 in scope (23 failing); 3 of those need more of the §19.8 streamability analysis | | RELAX NG | 100.00% — 965 of 965 | | Schemas wrongly refused | 7 — 6 on XSD 1.0, 1 on 1.1 | -| Tests | 2,887 `func Test` declarations, clean under `-race` | +| Tests | 2,907 `func Test` declarations, clean under `-race` | Every one of those failures, and why it is still open, is catalogued in [known-gaps.md](known-gaps.md). This file is the forward-looking half — what From 014a862163821e2fa5520cc28b1a90d80acd3216 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:20:08 -0400 Subject: [PATCH 56/98] xpath: bound deep-equal and map:find nesting at 1000 apart from function recursion --- xpath/context.go | 15 +++++++++++++ xpath/fn_31.go | 10 ++++----- xpath/fn_map.go | 5 ++--- xpath/valuedepth_test.go | 46 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 xpath/valuedepth_test.go diff --git a/xpath/context.go b/xpath/context.go index 9627a773..e3d0d567 100644 --- a/xpath/context.go +++ b/xpath/context.go @@ -62,6 +62,9 @@ type Context struct { // Depth guards against unbounded recursion in user-defined functions and // named templates, which the spec does not bound. Depth int32 + // valueDepth counts the nesting deep-equal and map:find have descended + // into, apart from Depth so that function recursion does not shrink it. + valueDepth int32 // heldItems suppresses Compiled.Eval's per-expression reset of items, // because a host language is measuring a larger evaluation against the // same counter. Set by HoldItemBudget, and copied along with the rest of @@ -1322,6 +1325,18 @@ func (c *Context) checkDepth() error { return nil } +// enterValue is checkDepth for the nesting of a map or array value, bounded +// at maxJSONDepth like parse-json and counted apart from function recursion. +// The caller decrements valueDepth when it returns. +func (c *Context) enterValue() error { + if c.valueDepth >= maxJSONDepth { + return fmt.Errorf("XPDY0001: value nests more than %d levels: %w", + maxJSONDepth, xdm.ErrResourceLimit) + } + c.valueDepth++ + return nil +} + // countItems charges n items against the evaluation budget. // // Accumulating constructs call it as they build, so a runaway is stopped while diff --git a/xpath/fn_31.go b/xpath/fn_31.go index 1a5a5cfe..47e9ea9a 100644 --- a/xpath/fn_31.go +++ b/xpath/fn_31.go @@ -1204,11 +1204,10 @@ func deepEqualMapArray(ctx *Context, x, y xdm.Item) (eq, handled bool, err error func deepEqualArrays(ctx *Context, a, b *xdm.ArrayItem) (bool, error) { // Nested values recurse here; bound them like any other recursion. - if err := ctx.checkDepth(); err != nil { + if err := ctx.enterValue(); err != nil { return false, err } - ctx.Depth++ - defer func() { ctx.Depth-- }() + defer func() { ctx.valueDepth-- }() if a.Len() != b.Len() { return false, nil } @@ -1227,11 +1226,10 @@ func deepEqualArrays(ctx *Context, a, b *xdm.ArrayItem) (bool, error) { func deepEqualMaps(ctx *Context, a, b *xdm.MapItem) (bool, error) { // Nested values recurse here; bound them like any other recursion. - if err := ctx.checkDepth(); err != nil { + if err := ctx.enterValue(); err != nil { return false, err } - ctx.Depth++ - defer func() { ctx.Depth-- }() + defer func() { ctx.valueDepth-- }() if a.Len() != b.Len() { return false, nil } diff --git a/xpath/fn_map.go b/xpath/fn_map.go index 99cef43c..5401f417 100644 --- a/xpath/fn_map.go +++ b/xpath/fn_map.go @@ -191,11 +191,10 @@ func registerMapFuncs(l *Library) { var walk func(seq xdm.Sequence) error walk = func(seq xdm.Sequence) error { // Recursion on the value's own nesting, so bounded like any other. - if err := ctx.checkDepth(); err != nil { + if err := ctx.enterValue(); err != nil { return err } - ctx.Depth++ - defer func() { ctx.Depth-- }() + defer func() { ctx.valueDepth-- }() for _, it := range seq { switch v := it.(type) { case *xdm.MapItem: diff --git a/xpath/valuedepth_test.go b/xpath/valuedepth_test.go new file mode 100644 index 00000000..567c4ee8 --- /dev/null +++ b/xpath/valuedepth_test.go @@ -0,0 +1,46 @@ +package xpath + +import ( + "errors" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// deep-equal and map:find bound value nesting at maxJSONDepth, as parse-json +// does, and count it apart from function recursion. +func TestValueNestingMatchesJSONBound(t *testing.T) { + nest := func(n int) xdm.Sequence { + v := xdm.One(xdm.NewInteger(1)) + for i := 0; i < n; i++ { + v = xdm.One(xdm.NewArray(v)) + } + return v + } + eval := func(query string, v xdm.Sequence, depth int32) error { + c, err := CompileWith(query, CompileOptions{Version: XPath31}) + if err != nil { + t.Fatal(err) + } + ctx := NewContext(nil, Builtins()).WithVar(xdm.QName{Local: "v"}, v) + ctx.Depth = depth + _, err = c.Eval(ctx) + return err + } + if err := eval(`deep-equal($v, $v)`, nest(maxJSONDepth), 0); err != nil { + t.Errorf("deep-equal at %d: %v", maxJSONDepth, err) + } + if err := eval(`deep-equal($v, $v)`, nest(maxJSONDepth), 400); err != nil { + t.Errorf("deep-equal at %d after 400 levels of recursion: %v", maxJSONDepth, err) + } + if err := eval(`deep-equal($v, $v)`, nest(maxJSONDepth+1), 0); !errors.Is(err, xdm.ErrResourceLimit) { + t.Errorf("deep-equal at %d: got %v, want a refusal", maxJSONDepth+1, err) + } + find := `Q{http://www.w3.org/2005/xpath-functions/map}find($v, 1)` + if err := eval(find, nest(maxJSONDepth-1), 400); err != nil { + t.Errorf("map:find at %d: %v", maxJSONDepth-1, err) + } + if err := eval(find, nest(maxJSONDepth+1), 0); !errors.Is(err, xdm.ErrResourceLimit) { + t.Errorf("map:find at %d: got %v, want a refusal", maxJSONDepth+1, err) + } +} From a9ad280b8d8e75a32a8af2719df7a7eddd1b3af2 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:20:24 -0400 Subject: [PATCH 57/98] xslt: join each run of adjacent atomics once instead of by repeated concatenation --- xslt/seqnorm.go | 25 +++++++++++++--------- xslt/seqnorm_test.go | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 xslt/seqnorm_test.go diff --git a/xslt/seqnorm.go b/xslt/seqnorm.go index c3be307b..0ce91a7b 100644 --- a/xslt/seqnorm.go +++ b/xslt/seqnorm.go @@ -1,6 +1,10 @@ package xslt -import "github.com/knroy/go-xml/v2/xdm" +import ( + "strings" + + "github.com/knroy/go-xml/v2/xdm" +) // joinAdjacentAtomics is step 1 of sequence normalisation (XSLT and XQuery // Serialization 3.1, section 2): every maximal run of adjacent atomic values @@ -26,22 +30,23 @@ func joinAdjacentAtomics(seq xdm.Sequence) xdm.Sequence { return seq } out := make(xdm.Sequence, 0, len(seq)) - prevAtomic := false + var run []string + flush := func() { + if len(run) > 0 { + out = append(out, xdm.NewNode(xdm.KindText, xdm.QName{}, strings.Join(run, " "))) + run = run[:0] + } + } for _, it := range seq { a, ok := it.(*xdm.Atomic) if !ok { + flush() out = append(out, it) - prevAtomic = false - continue - } - if prevAtomic { - last := out[len(out)-1].(*xdm.Node) - last.SetValue(last.Value() + " " + a.String()) continue } - out = append(out, xdm.NewNode(xdm.KindText, xdm.QName{}, a.String())) - prevAtomic = true + run = append(run, a.String()) } + flush() return out } diff --git a/xslt/seqnorm_test.go b/xslt/seqnorm_test.go new file mode 100644 index 00000000..c2c43d91 --- /dev/null +++ b/xslt/seqnorm_test.go @@ -0,0 +1,50 @@ +package xslt + +import ( + goruntime "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// A run of n adjacent atomics used to be joined by repeated concatenation, +// which copies the growing string n times and allocates O(n^2) bytes. +func TestJoinAdjacentAtomicsIsLinear(t *testing.T) { + const n = 20000 + var seq xdm.Sequence + seq = append(seq, xdm.NewString("a")) + for i := 1; i < n; i++ { + seq = append(seq, xdm.NewInteger(int64(i))) + } + elem := xdm.NewNode(xdm.KindElement, xdm.QName{Local: "e"}, "") + seq = append(seq, elem, xdm.NewString("x"), xdm.NewString("y")) + + var before, after goruntime.MemStats + goruntime.ReadMemStats(&before) + out := joinAdjacentAtomics(seq) + goruntime.ReadMemStats(&after) + + if len(out) != 3 { + t.Fatalf("got %d items, want 3", len(out)) + } + var want strings.Builder + want.WriteString("a") + for i := 1; i < n; i++ { + want.WriteString(" " + xdm.NewInteger(int64(i)).String()) + } + if got := out[0].(*xdm.Node).Value(); got != want.String() { + t.Fatalf("first run differs: len %d, want %d", len(got), want.Len()) + } + if out[1] != xdm.Item(elem) { + t.Fatalf("node item not preserved") + } + if got := out[2].(*xdm.Node).Value(); got != "x y" { + t.Fatalf("second run = %q", got) + } + // Linear joining allocates a few hundred KB here; the quadratic form + // allocated about 1 GB. + if d := after.TotalAlloc - before.TotalAlloc; d > 20<<20 { + t.Fatalf("joinAdjacentAtomics allocated %d bytes for %d atomics", d, n) + } +} From b35ec14e6bcb56d5975c2b6335c6de87e1bb2022 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:20:27 -0400 Subject: [PATCH 58/98] xslt: charge xsl:copy-of of a document node against MaxNodes --- xslt/instructions.go | 6 ++++++ xslt/nodebudget_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/xslt/instructions.go b/xslt/instructions.go index 165805a4..5f938910 100644 --- a/xslt/instructions.go +++ b/xslt/instructions.go @@ -220,6 +220,12 @@ func (i *copyOfInstr) Execute(rt *runtime, out *outputBuilder) error { // destinations that keep a bare sequence — a variable // declared as="document-node()*", a function body — must see // the wrapper itself. + // Charged before the copy is made, as the node branch below + // does; copy-of of a 100 MB document copied 5.7M nodes + // uncharged while shallow-copy of it was refused. + if err := rt.ctx.ChargeNodes(countSubtree(v)); err != nil { + return err + } c := copyDocumentNode(v) if i.noNamespaces { // §11.9.2's namespace rules are about the elements being diff --git a/xslt/nodebudget_test.go b/xslt/nodebudget_test.go index e9085920..0032db9a 100644 --- a/xslt/nodebudget_test.go +++ b/xslt/nodebudget_test.go @@ -380,3 +380,27 @@ func TestInternBindingsHitDoesNotAllocate(t *testing.T) { t.Errorf("an interning hit allocates %v times, want 0", n) } } + +// xsl:copy-of of a document node was the one copy the node budget never saw: +// the node branch charges, the document branch did not, so copy-of select="/" +// of a 2.1 million node document exited 0 while shallow-copy of it was refused. +func TestCopyOfDocumentNodeIsChargedAgainstMaxNodes(t *testing.T) { + sd, err := xdm.ParseString(``+ + ``+ + ``, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + st, err := Compile(sd.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + doc, err := xdm.ParseString(""+strings.Repeat(`t`, 700000)+"", + xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + _, err = st.Transform(context.Background(), doc.Root, TransformOptions{}) + wantNodeRefusal(t, err) +} From 676ad52abe61f09d44d136d96f2766d56445b7c7 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:20:42 -0400 Subject: [PATCH 59/98] relaxng: publish pending nameMemo entries after a few locked hits --- relaxng/namememo_test.go | 20 +++++++++++++++++++ relaxng/pattern.go | 42 ++++++++++++++++++++++++++++++---------- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/relaxng/namememo_test.go b/relaxng/namememo_test.go index d68b033e..4502b927 100644 --- a/relaxng/namememo_test.go +++ b/relaxng/namememo_test.go @@ -32,3 +32,23 @@ func TestNameMemoStoreAmortised(t *testing.T) { t.Errorf("%d entries, cap is %d", n, maxOpenMemo) } } + +// Pending entries are published after a few locked hits, so a steady +// lookup stream becomes lock-free instead of staying on the slow path. +func TestNameMemoPendingPublishedOnHits(t *testing.T) { + var c nameMemo + for _, n := range []string{"a", "b"} { // 2nd store leaves one pending + c.store(xdm.QName{Local: n}, emptyPat{}) + } + if c.np.Load() == 0 { + t.Fatal("setup: expected a pending entry") + } + for i := 0; i < pendingHitsPublish; i++ { + if _, ok := c.load(xdm.QName{Local: "b"}); !ok { + t.Fatal("pending entry lost") + } + } + if c.np.Load() != 0 || len(*c.m.Load()) != 2 { + t.Errorf("pending not published: np=%d snapshot=%d", c.np.Load(), len(*c.m.Load())) + } +} diff --git a/relaxng/pattern.go b/relaxng/pattern.go index 67894487..df99bffc 100644 --- a/relaxng/pattern.go +++ b/relaxng/pattern.go @@ -184,6 +184,7 @@ type nameMemo struct { mu sync.Mutex pending map[nameKey]pattern // stored but not yet published in m; under mu np atomic.Int32 // len(pending), so a load skips the lock when empty + hits int // locked pending hits since the last publish; under mu } type nameKey struct{ ns, local string } @@ -200,10 +201,21 @@ func (c *nameMemo) load(n xdm.QName) (pattern, bool) { } c.mu.Lock() p, ok := c.pending[k] + // A pending hit costs a lock; after a few, publish so later lookups + // are lock-free again. + if ok { + if c.hits++; c.hits >= pendingHitsPublish { + c.publishLocked() + } + } c.mu.Unlock() return p, ok } +// pendingHitsPublish is how many locked lookups of pending entries trigger +// their publication. +const pendingHitsPublish = 8 + func (c *nameMemo) store(n xdm.QName, p pattern) { c.mu.Lock() defer c.mu.Unlock() @@ -226,17 +238,27 @@ func (c *nameMemo) store(n xdm.QName, p pattern) { // sum to O(n) over n stores. A load that misses the snapshot checks the // pending map. if len(c.pending) > len(old) { - m := make(map[nameKey]pattern, len(old)+len(c.pending)) - for k, v := range old { - m[k] = v - } - for k, v := range c.pending { - m[k] = v - } - c.m.Store(&m) - c.pending = nil - c.np.Store(0) + c.publishLocked() + } +} + +// publishLocked folds the pending entries into a new snapshot. +func (c *nameMemo) publishLocked() { + var old map[nameKey]pattern + if o := c.m.Load(); o != nil { + old = *o + } + m := make(map[nameKey]pattern, len(old)+len(c.pending)) + for k, v := range old { + m[k] = v + } + for k, v := range c.pending { + m[k] = v } + c.m.Store(&m) + c.pending = nil + c.np.Store(0) + c.hits = 0 } type patBox struct { From 331bc96ad877d58cb1cc484a756b07ba7157f173 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:21:24 -0400 Subject: [PATCH 60/98] xpath: do not cache a compiled regex over 4,096 instructions --- xpath/fn_regex.go | 7 +++++++ xpath/regex_cache_size_test.go | 28 ++++++++++++++++++++++++++++ xpath/regex_size.go | 11 +++++++++++ 3 files changed, 46 insertions(+) create mode 100644 xpath/regex_cache_size_test.go diff --git a/xpath/fn_regex.go b/xpath/fn_regex.go index f49d5428..8ef0651a 100644 --- a/xpath/fn_regex.go +++ b/xpath/fn_regex.go @@ -312,6 +312,13 @@ func storeRegex(key string, v any) { if len(key) > maxCachedKey { return } + // A pattern under the key limit can still compile to megabytes (85 copies + // of \p{L}{1000} is about 1 KB and retains 3.95 MB), so a large program is + // not kept either. The largest real pattern in the corpora is about 2,593 + // instructions. + if re, ok := v.(*regexp.Regexp); ok && cachedProgramTooBig(re) { + return + } regexCache.Put(key, v) } diff --git a/xpath/regex_cache_size_test.go b/xpath/regex_cache_size_test.go new file mode 100644 index 00000000..b856bff8 --- /dev/null +++ b/xpath/regex_cache_size_test.go @@ -0,0 +1,28 @@ +package xpath + +import ( + "strings" + "testing" +) + +// A pattern under the key limit can still compile to megabytes, so the cache +// keeps a program only up to maxCachedProgram instructions. +func TestLargeProgramIsNotCached(t *testing.T) { + big := strings.Repeat(`\p{L}{1000}`, 40) + "x" + if len(big) > maxCachedKey { + t.Fatalf("test pattern is %d bytes, over the key limit", len(big)) + } + n := regexCache.Len() + if _, err := compileXPathRegexp(big, "", XPath20); err != nil { + t.Fatal(err) + } + if regexCache.Len() != n { + t.Errorf("a %d-byte pattern with a huge program was cached", len(big)) + } + if _, err := compileXPathRegexp(`[a-z]+\d{3}`, "", XPath20); err != nil { + t.Fatal(err) + } + if regexCache.Len() != n+1 { + t.Errorf("a small pattern was not cached") + } +} diff --git a/xpath/regex_size.go b/xpath/regex_size.go index 9dce43a2..aaa831d5 100644 --- a/xpath/regex_size.go +++ b/xpath/regex_size.go @@ -58,3 +58,14 @@ func progSize(re *syntax.Regexp, memo map[*syntax.Regexp]int) int { memo[re] = n return n } + +// maxCachedProgram is the largest program, in estimated RE2 instructions, that +// the regex cache will retain. +const maxCachedProgram = 4096 + +// cachedProgramTooBig reports whether re is too large to cache. It re-parses +// the compiled source, which is cheap beside the compile that just ran. +func cachedProgramTooBig(re *regexp.Regexp) bool { + p, err := syntax.Parse(re.String(), syntax.Perl) + return err == nil && progSize(p.Simplify(), map[*syntax.Regexp]int{}) > maxCachedProgram +} From dcb49b26da8ca3fb51cde261caa88dd48d9e9209 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:21:30 -0400 Subject: [PATCH 61/98] xpath: charge the copies a sequence constructor and ! make --- xpath/copycharge_test.go | 32 ++++++++++++++++++++++++++++++++ xpath/eval.go | 20 ++++++++++++++++++++ xpath/parser.go | 5 +++++ 3 files changed, 57 insertions(+) create mode 100644 xpath/copycharge_test.go diff --git a/xpath/copycharge_test.go b/xpath/copycharge_test.go new file mode 100644 index 00000000..85b16142 --- /dev/null +++ b/xpath/copycharge_test.go @@ -0,0 +1,32 @@ +package xpath + +import ( + "strings" + "testing" +) + +// A sequence constructor or a "!" that copies its operand into a larger one +// is charged for the copies, so doubling it in a fold hits the item budget. +// The 23 doublings build 16.7M items against the 5M limit. +func TestSequenceCopiesAreCharged(t *testing.T) { + for _, q := range []string{ + `fold-left(1 to 23, (1,2), function($s,$i){($s,$s)})`, + `fold-left(1 to 23, (1,2), function($s,$i){$s ! (.,.)})`, + } { + c, err := CompileWith(q, CompileOptions{Version: XPath31}) + if err != nil { + t.Fatal(err) + } + _, err = c.Eval(NewContext(nil, Builtins())) + if err == nil || !strings.Contains(err.Error(), "XPDY0130") { + t.Errorf("%s: got %v, want XPDY0130", q, err) + } + } + // A legal modest sequence still evaluates, with and without the floor. + for _, q := range []string{`count((1 to 1000, 1 to 1000))`, `count(((1,2),(3,4)))`, `count((1 to 1000) ! (., .))`} { + c, _ := CompileWith(q, CompileOptions{Version: XPath31}) + if _, err := c.Eval(NewContext(nil, Builtins())); err != nil { + t.Errorf("%s: %v", q, err) + } + } +} diff --git a/xpath/eval.go b/xpath/eval.go index bb57c673..10f799ba 100644 --- a/xpath/eval.go +++ b/xpath/eval.go @@ -42,16 +42,36 @@ func (e *ContextItem) Eval(ctx *Context) (xdm.Sequence, error) { // Eval implements Expr for a sequence constructor. func (e *SequenceExpr) Eval(ctx *Context) (xdm.Sequence, error) { var out xdm.Sequence + widest := 0 for _, it := range e.Items { v, err := it.Eval(ctx) if err != nil { return nil, err } + widest = max(widest, len(v)) out = append(out, v...) } + // The widest operand was charged by whatever built it; the rest is + // copies, which ($s, $s) in a loop doubles without any other charge. + if err := ctx.chargeCopies(len(out) - widest); err != nil { + return nil, err + } return out, nil } +// chargeCopyFloor is the size below which chargeCopies charges nothing, so +// that ordinary small sequences cost no atomic add. +const chargeCopyFloor = 64 + +// chargeCopies charges n items a constructor copied beyond those the +// expressions that built them already charged. +func (c *Context) chargeCopies(n int) error { + if n < chargeCopyFloor { + return nil + } + return c.countItems(n) +} + // Eval implements Expr for a single axis step. // // A step is evaluated against the context item alone; iterating a step over diff --git a/xpath/parser.go b/xpath/parser.go index 7bf5b7a8..bb060401 100644 --- a/xpath/parser.go +++ b/xpath/parser.go @@ -1274,6 +1274,11 @@ func (e *SimpleMap) Eval(ctx *Context) (xdm.Sequence, error) { } out = append(out, v...) } + // One result per input is covered by the input's own charge; more than + // that is copying, which a doubling right-hand side compounds. + if err := ctx.chargeCopies(len(out) - len(in)); err != nil { + return nil, err + } return out, nil } From 15273c79e722bf0d703929b165488495b0853084 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:22:13 -0400 Subject: [PATCH 62/98] xslt: memoise function results under a 128-bit hash, not the key string --- xslt/apply.go | 27 +++++++++++++++++++++++---- xslt/funcmemo_test.go | 32 ++++++++++++++++++++++++++++++++ xslt/runtime.go | 4 ++-- 3 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 xslt/funcmemo_test.go diff --git a/xslt/apply.go b/xslt/apply.go index bad01a66..4c5964ed 100644 --- a/xslt/apply.go +++ b/xslt/apply.go @@ -1,6 +1,7 @@ package xslt import ( + "crypto/sha256" "errors" "fmt" "sort" @@ -766,7 +767,8 @@ func (f *userFunction) call(ctx *xpath.Context, args []xdm.Sequence) (xdm.Sequen // declares new-each-time="yes" and requires 10, and function-1029 // declares "maybe" and accepts anything from 7 to 10 -- so the caching // must be keyed on the setting and not merely be an optimisation. - var cacheKey string + var cacheKey callHash + var haveKey bool if f.deterministic { switch k, err := rt.functionCallKey(f, args); { case errors.Is(err, errNotCacheable): @@ -775,10 +777,11 @@ func (f *userFunction) call(ctx *xpath.Context, args []xdm.Sequence) (xdm.Sequen case err != nil: return nil, err default: - if hit, ok := rt.funcResults[string(k)]; ok { + cacheKey = hashCallKey(k) + haveKey = true + if hit, ok := rt.funcResults[cacheKey]; ok { return hit, nil } - cacheKey = string(k) } } @@ -849,7 +852,7 @@ func (f *userFunction) call(ctx *xpath.Context, args []xdm.Sequence) (xdm.Sequen if err != nil { return nil, err } - if f.deterministic && cacheKey != "" { + if f.deterministic && haveKey { // Only a successful call is remembered. An error is not a result, and // caching one would make a failure that depended on transient state // -- an unavailable document, say -- permanent for the rest of the @@ -859,6 +862,22 @@ func (f *userFunction) call(ctx *xpath.Context, args []xdm.Sequence) (xdm.Sequen return res, nil } +// callHash is the first 128 bits of the SHA-256 of a functionCallKey. The memo +// keeps this instead of the key itself, which for a map or array argument is +// its whole content (about 300 B a key on DocBook table-cals.049). +// +// A collision would return another call's result, and nothing compares the +// full key on a hit. That is accepted: SHA-256 is collision resistant, so an +// input cannot be crafted to collide (MD5 or a 64-bit hash would allow it), +// and by chance two keys in one transform collide with probability about +// n^2/2^129, below 1e-20 for a billion calls. +type callHash [16]byte + +func hashCallKey(k []byte) callHash { + h := sha256.Sum256(k) + return callHash(h[:16]) +} + // maxKeyBufCap is the largest scratch buffer functionCallKey keeps: 64 KB. const maxKeyBufCap = 64 << 10 diff --git a/xslt/funcmemo_test.go b/xslt/funcmemo_test.go new file mode 100644 index 00000000..beadaedf --- /dev/null +++ b/xslt/funcmemo_test.go @@ -0,0 +1,32 @@ +package xslt + +import ( + "context" + "testing" + "unsafe" + + "github.com/knroy/go-xml/v2/xdm" +) + +// The function memo keeps a 16-byte hash of each call's key, not the key. On +// DocBook table-cals.049 the keys of map and array arguments were 209 MB of a +// 365 MB live heap. +func TestFunctionMemoKeepsHashNotKey(t *testing.T) { + s := compileString(t, ``) + rt, err := newRuntime(s, context.Background(), nil, TransformOptions{}) + if err != nil { + t.Fatal(err) + } + var k callHash + if got := unsafe.Sizeof(k); got != 16 { + t.Errorf("memo key is %d bytes, want 16", got) + } + rt.funcResults[hashCallKey([]byte("a"))] = xdm.Sequence{} + if _, ok := rt.funcResults[hashCallKey([]byte("b"))]; ok { + t.Error("distinct keys share a hash") + } + if _, ok := rt.funcResults[hashCallKey([]byte("a"))]; !ok { + t.Error("equal keys do not share a hash") + } +} diff --git a/xslt/runtime.go b/xslt/runtime.go index f0c01b89..05d3cff1 100644 --- a/xslt/runtime.go +++ b/xslt/runtime.go @@ -164,7 +164,7 @@ type transformState struct { // building nodes means the same nodes rather than merely equal ones -- // so the promise is only kept by evaluating once and reusing. Keyed by // functionCallKey; see apply.go. - funcResults map[string]xdm.Sequence + funcResults map[callHash]xdm.Sequence // itemIDs numbers the nodes and function items functionCallKey keys by // identity; see identityNumber. @@ -759,7 +759,7 @@ func newRuntime(s *Stylesheet, ctx context.Context, root *xdm.Node, opts Transfo accumOrigin: map[*xdm.Node]*xdm.Node{}, treeAccums: map[*xdm.Node]*modeAccumulators{}, streamedTrees: map[*xdm.Node]bool{}, - funcResults: map[string]xdm.Sequence{}, + funcResults: map[callHash]xdm.Sequence{}, itemIDs: map[any]int{}, messages: new([]string), warnings: new([]string), From 40cd52ccee9f126ddd6ace7ddc39c298de7c3928 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:22:14 -0400 Subject: [PATCH 63/98] xslt: share and charge sequence-valued accumulator values --- xslt/accumshare_test.go | 61 +++++++++++++++++++++++++++++++++++++++++ xslt/accumulator.go | 59 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 xslt/accumshare_test.go diff --git a/xslt/accumshare_test.go b/xslt/accumshare_test.go new file mode 100644 index 00000000..c573dddd --- /dev/null +++ b/xslt/accumshare_test.go @@ -0,0 +1,61 @@ +package xslt + +import ( + "context" + "errors" + "strconv" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// A sequence-valued accumulator, ($value, 1), used to keep a full copy of its +// value at every node: n nodes held n^2/2 items, none of them charged. Now a +// value that extends its predecessor shares its backing array, and what is +// stored counts against MaxItems. +func TestAccumulatorSequenceValueIsChargedAndShared(t *testing.T) { + const sheet = ` + + + + + + + ` + run := func(n, max int) (string, error) { + st, err := xdm.ParseString(sheet, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + c, err := Compile(st.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + src, err := xdm.ParseString(""+strings.Repeat("", n)+"", xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + res, err := c.Transform(context.Background(), src.Root, TransformOptions{MaxItems: max}) + if err != nil { + return "", err + } + var b strings.Builder + if err := res.Serialize(&b); err != nil { + return "", err + } + return b.String(), nil + } + // The stored items number n, so a bound of n+100 accepts the walk and + // the output is unchanged. + if got, err := run(2000, 2100); err != nil || got != strconv.Itoa(2000) { + t.Fatalf("n=2000: got %q, %v", got, err) + } + // A bound just under n refuses it. + _, err := run(2000, 1900) + if err == nil || !errors.Is(err, xdm.ErrResourceLimit) { + t.Fatalf("n=2000 with MaxItems 1900: err = %v, want a resource limit", err) + } +} diff --git a/xslt/accumulator.go b/xslt/accumulator.go index abec5a88..efdd83ec 100644 --- a/xslt/accumulator.go +++ b/xslt/accumulator.go @@ -369,13 +369,45 @@ func (rt *runtime) accumulatorValuesFor(def *accumulatorDef, root *xdm.Node, // The walk is pre/post order over the whole tree: every node records the // value on the way in and the value on the way out, and the start and end // rules that match it move the value between the two. + // + // A sequence-valued accumulator ($value, 1) used to leave every node its + // own full copy, so n nodes held n^2/2 items. extend lets a value that + // grows by appending share its predecessor's backing array, and held + // counts what is really stored, one entry per item beyond the first, so + // that a document of single-item values is not refused by an item bound + // that was meant for sequences. + var tip xdm.Sequence + var held int64 + extend := func(prev, next xdm.Sequence) (xdm.Sequence, error) { + if len(next) == len(prev) && (len(next) == 0 || &next[0] == &prev[0]) { + return next, nil + } + k := len(prev) + if k > 0 && len(next) > k && len(tip) == k && &tip[0] == &prev[0] && + sameItems(prev, next[:k]) { + held += int64(len(next) - k) + next = append(prev, next[k:]...) + tip = next + } else { + if len(next) > 1 { + held += int64(len(next)) + } + // Clamped so that a later append reallocates instead of + // writing into spare room the rule's value may share. + next = next[:len(next):len(next)] + tip = next + } + return next, ctx.ItemsRefused(held) + } var walk func(*xdm.Node) error walk = func(n *xdm.Node) error { next, err := rt.applyAccumRules(def, n, cur, ctx, false) if err != nil { return err } - cur = next + if cur, err = extend(cur, next); err != nil { + return err + } vals.before[n] = cur // Attributes and namespaces are NOT visited. §18.2.4's formal model // defines an accumulator over the sequence of events a streamed @@ -403,7 +435,9 @@ func (rt *runtime) accumulatorValuesFor(def *accumulatorDef, root *xdm.Node, if err != nil { return err } - cur = end + if cur, err = extend(cur, end); err != nil { + return err + } vals.after[n] = cur return nil } @@ -415,6 +449,27 @@ func (rt *runtime) accumulatorValuesFor(def *accumulatorDef, root *xdm.Node, return vals, nil } +// sameItems reports whether a and b hold the identical items. Only nodes and +// atomic values are compared, by identity; any other item makes the answer no, +// which merely costs the sharing. +func sameItems(a, b xdm.Sequence) bool { + for i, x := range a { + switch x := x.(type) { + case *xdm.Atomic: + if y, ok := b[i].(*xdm.Atomic); !ok || x != y { + return false + } + case *xdm.Node: + if y, ok := b[i].(*xdm.Node); !ok || x != y { + return false + } + default: + return false + } + } + return true +} + // applyAccumRules runs the highest-priority rule of the requested phase that // matches n, and returns the value it leaves behind. // From 5951f24842eda37011537a1181bed791e8a1f697 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:22:30 -0400 Subject: [PATCH 64/98] xpath: bind a dynamic call's host names in one scope each, not one per name --- xpath/clearhostvars_test.go | 36 +++++++++++++++++++++++++++++++++ xpath/context.go | 40 +++++++++++++++++++++++++++++++++++++ xpath/funcitem.go | 13 +++--------- 3 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 xpath/clearhostvars_test.go diff --git a/xpath/clearhostvars_test.go b/xpath/clearhostvars_test.go new file mode 100644 index 00000000..39729c54 --- /dev/null +++ b/xpath/clearhostvars_test.go @@ -0,0 +1,36 @@ +package xpath + +import ( + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// TestClearHostVarsOneScope: the host's cleared and marked names are bound in +// one scope each, not one scope per name, and still resolve as before. +func TestClearHostVarsOneScope(t *testing.T) { + oc, om := ClearedOnDynamicCall, MarkedOnDynamicCall + defer func() { ClearedOnDynamicCall, MarkedOnDynamicCall = oc, om }() + var cleared []xdm.QName + for _, n := range []string{"a", "b", "c", "d", "e", "f"} { + cleared = append(cleared, xdm.QName{URI: "urn:h", Local: n}) + } + ClearedOnDynamicCall = cleared + MarkedOnDynamicCall = []xdm.QName{{URI: "urn:h", Local: "m"}} + + base := NewContext(nil, Builtins()).WithVar(xdm.QName{URI: "urn:h", Local: "c"}, xdm.One(xdm.NewInteger(7))) + got := clearHostVars(base) + if v, ok := got.LookupVar(cleared[2]); !ok || len(v) != 0 { + t.Fatalf("cleared name resolves to %v, %v; want bound and empty", v, ok) + } + if v, ok := got.LookupVar(MarkedOnDynamicCall[0]); !ok || len(v) != 1 { + t.Fatalf("marked name resolves to %v, %v; want one true", v, ok) + } + depth := 0 + for s := got; s.bind != base.bind; s = s.Parent { + depth++ + } + if depth != 2 { + t.Fatalf("clearHostVars made %d scopes, want 2", depth) + } +} diff --git a/xpath/context.go b/xpath/context.go index 9627a773..32220630 100644 --- a/xpath/context.go +++ b/xpath/context.go @@ -922,6 +922,35 @@ func (c *Context) WithVar(name xdm.QName, val xdm.Sequence) *Context { return n } +// withVars binds every name to val in one scope. The names have non-empty +// local parts; it is what the host registries of a dynamic call are bound with. +func (c *Context) withVars(names []xdm.QName, val xdm.Sequence) *Context { + if len(names) == 0 { + return c + } + s := &struct { + n Context + b varBinding + }{n: *c} + s.n.Vars = nil + s.b = varBinding{names: names, val: val} + s.n.bind = &s.b + s.n.Parent = c + return &s.n +} + +// WithVars is WithVar for several variables at once: one scope binds names[i] +// to vals[i]. The names must be distinct, non-empty and as long as vals, and +// neither slice may be written while the scope is in use. +func (c *Context) WithVars(names []xdm.QName, vals []xdm.Sequence) *Context { + if len(names) == 0 { + return c + } + n := c.withVars(names, nil) + n.bind.vals = vals + return n +} + // varBinding is the binding a scope made by WithVar adds. Never written once // the scope is published. A scope made by WithLazyVars holds lazy instead, // and binds every name in it; one made by xpathleaf.WithLocals holds locals, @@ -929,6 +958,8 @@ func (c *Context) WithVar(name xdm.QName, val xdm.Sequence) *Context { type varBinding struct { uri, local string val xdm.Sequence + names []xdm.QName // withVars: every name is bound, to vals[i] or else to val + vals []xdm.Sequence lazy map[varKey]*LazyVar locals *xpathleaf.Locals } @@ -1240,6 +1271,15 @@ func (c *Context) lookupVarPlain(name xdm.QName) (xdm.Sequence, bool, error) { return v, true, err } } + } else if b.names != nil { + for i, q := range b.names { + if q.Local == name.Local && q.URI == name.URI { + if b.vals != nil { + return b.vals[i], true, nil + } + return b.val, true, nil + } + } } else if b.local == name.Local && b.uri == name.URI { return b.val, true, nil } diff --git a/xpath/funcitem.go b/xpath/funcitem.go index 3360e04e..0309b552 100644 --- a/xpath/funcitem.go +++ b/xpath/funcitem.go @@ -167,9 +167,7 @@ func withRetainedFocus(ref *Context, inner func(any, []xdm.Sequence) (xdm.Sequen // names regex-group#1(2) as the example -- the components are // cleared, so regex-090 gets the zero-length string rather than // the enclosing match. - for _, name := range ClearedOnDynamicCall { - p = p.WithVar(name, nil) - } + p = p.withVars(ClearedOnDynamicCall, nil) // The host state is a binding too: it comes from the call, with // the current item cleared as the variables above are. p.host = hostOnCall(c.host, false) @@ -643,13 +641,8 @@ func clearHostVars(ctx *Context) *Context { n.host = h ctx = &n } - for _, name := range ClearedOnDynamicCall { - ctx = ctx.WithVar(name, nil) - } - for _, name := range MarkedOnDynamicCall { - ctx = ctx.WithVar(name, xdm.One(xdm.NewBoolean(true))) - } - return ctx + ctx = ctx.withVars(ClearedOnDynamicCall, nil) + return ctx.withVars(MarkedOnDynamicCall, xdm.One(xdm.NewBoolean(true))) } // Eval implements Expr: it calls the function item the target produces. From a4f316d63165bcd17c3da38958e6f809d9fe4ad4 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:22:30 -0400 Subject: [PATCH 65/98] xquery: bind a tuple and a window condition's variables in one scope --- xquery/flwor.go | 6 +--- xquery/window.go | 69 +++++++++++++++++++++++++++++-------- xquery/window_alloc_test.go | 30 ++++++++++++++++ 3 files changed, 86 insertions(+), 19 deletions(-) create mode 100644 xquery/window_alloc_test.go diff --git a/xquery/flwor.go b/xquery/flwor.go index 40e5e617..b4279108 100644 --- a/xquery/flwor.go +++ b/xquery/flwor.go @@ -112,11 +112,7 @@ func (t tuple) lookup(name xdm.QName) (xdm.Sequence, bool) { // than at parse, so nothing about the names has to reach the expression // parser: binding the values here is the whole of the scoping. func (t tuple) context(ctx *evalContext) *xpath.Context { - xp := ctx.xp - for i, n := range t.names { - xp = xp.WithVar(n, t.vals[i]) - } - return xp + return ctx.xp.WithVars(t.names, t.vals) } // sub is context wrapped back up as an evalContext, which is what a diff --git a/xquery/window.go b/xquery/window.go index 40fb2db2..a18bc5a3 100644 --- a/xquery/window.go +++ b/xquery/window.go @@ -1,6 +1,8 @@ package xquery import ( + "slices" + "github.com/knroy/go-xml/v2/xdm" ) @@ -26,30 +28,68 @@ type windowVars struct { // §3.10.4 defines "previous" and "next" as the empty sequence at the ends of // the sequence rather than as an error, which is what makes // "end $x next $y when empty($y)" the idiom for "the last window". -func (v *windowVars) bind(t tuple, seq xdm.Sequence, i int) tuple { +func (v *windowVars) bind(t tuple, seq xdm.Sequence, i int, x *nameExt) tuple { + // The names this condition binds are laid out once and added to the tuple + // in one pass, sharing the extended names slice through x, rather than + // copying both slices once per variable. + var nm [4]xdm.QName + var vs [4]xdm.Sequence + k := 0 if v.hasItem { - t = t.bind(v.item, xdm.One(seq[i-1])) + nm[k], vs[k] = v.item, xdm.One(seq[i-1]) + k++ } if v.hasPos { - t = t.bind(v.pos, xdm.One(xdm.NewInteger(int64(i)))) + nm[k], vs[k] = v.pos, xdm.One(xdm.NewInteger(int64(i))) + k++ } if v.hasPrev { var prev xdm.Sequence if i >= 2 { prev = xdm.One(seq[i-2]) } - t = t.bind(v.prev, prev) + nm[k], vs[k] = v.prev, prev + k++ } if v.hasNext { var next xdm.Sequence if i < len(seq) { next = xdm.One(seq[i]) } - t = t.bind(v.next, next) + nm[k], vs[k] = v.next, next + k++ + } + for _, n := range nm[:k] { + if slices.Contains(t.names, n) { // rebinding replaces; the slow path + for j := 0; j < k; j++ { + t = t.bind(nm[j], vs[j]) + } + return t + } + } + if k == 0 { + return t } - return t + vals := make([]xdm.Sequence, len(t.vals)+k) + copy(vals, t.vals) + copy(vals[len(t.vals):], vs[:k]) + return tuple{names: x.extendN(t.names, nm[:k]), vals: vals} } +// extendN is extend for several names at once. +func (x *nameExt) extendN(names, add []xdm.QName) []xdm.QName { + if x.to == nil || len(names) != len(x.from) || + (len(names) > 0 && &names[0] != &x.from[0]) { + x.from = names + x.to = append(append(make([]xdm.QName, 0, len(names)+len(add)), names...), add...) + } + return x.to +} + +// winExt is the names slices a windows call extends: the start condition's on +// the input tuple, and the end condition's. +type winExt struct{ start, end nameExt } + // names returns every variable this condition binds, which is what the // distinctness rule of XQST0103 is checked over. func (v *windowVars) names() []xdm.QName { @@ -126,8 +166,9 @@ func (c *windowClause) apply(in []tuple, ctx *evalContext) ([]tuple, error) { // window. func (c *windowClause) windows(t tuple, seq xdm.Sequence, ctx *evalContext) ([]tuple, error) { var out []tuple + var x winExt for s := 1; s <= len(seq); s++ { - st := c.start.bind(t, seq, s) + st := c.start.bind(t, seq, s, &x.start) ok, err := c.start.when.evalBool(st.sub(ctx)) if err != nil { return nil, err @@ -135,7 +176,7 @@ func (c *windowClause) windows(t tuple, seq xdm.Sequence, ctx *evalContext) ([]t if !ok { continue } - end, closed, err := c.findEnd(st, seq, s, ctx) + end, closed, err := c.findEnd(st, seq, s, ctx, &x) if err != nil { return nil, err } @@ -149,7 +190,7 @@ func (c *windowClause) windows(t tuple, seq xdm.Sequence, ctx *evalContext) ([]t // where the specification asks for four. continue } - w := c.emit(st, seq, s, end, closed) + w := c.emit(st, seq, s, end, closed, &x) if c.typeCheck != nil { v, _ := w.lookup(c.name) if _, err := applyCheck(c.typeCheck, v, ctx); err != nil { @@ -173,10 +214,10 @@ func (c *windowClause) windows(t tuple, seq xdm.Sequence, ctx *evalContext) ([]t // start, so the search is for the next position that satisfies the *start* // condition — the one case where the end is defined in terms of the start. func (c *windowClause) findEnd(st tuple, seq xdm.Sequence, s int, - ctx *evalContext) (int, bool, error) { + ctx *evalContext, x *winExt) (int, bool, error) { if !c.hasEnd { for e := s + 1; e <= len(seq); e++ { - nt := c.start.bind(st, seq, e) + nt := c.start.bind(st, seq, e, &x.start) ok, err := c.start.when.evalBool(nt.sub(ctx)) if err != nil { return 0, false, err @@ -188,7 +229,7 @@ func (c *windowClause) findEnd(st tuple, seq xdm.Sequence, s int, return len(seq), false, nil } for e := s; e <= len(seq); e++ { - et := c.end.bind(st, seq, e) + et := c.end.bind(st, seq, e, &x.end) ok, err := c.end.when.evalBool(et.sub(ctx)) if err != nil { return 0, false, err @@ -204,7 +245,7 @@ func (c *windowClause) findEnd(st tuple, seq xdm.Sequence, s int, // items, and every boundary variable bound to the value it had at the // boundary. func (c *windowClause) emit(st tuple, seq xdm.Sequence, s, e int, - closed bool) tuple { + closed bool, x *winExt) tuple { t := st // §3.10.4 binds the end variables to the window's last item whether the // end condition fired there or the window simply ran out of sequence. @@ -216,7 +257,7 @@ func (c *windowClause) emit(st tuple, seq xdm.Sequence, s, e int, // An empty window has no last item, so there is nothing to bind and the // variables stay unbound rather than being bound to nothing. if (c.hasEnd || !c.sliding) && e >= s { - t = c.end.bind(t, seq, e) + t = c.end.bind(t, seq, e, &x.end) } return t.bind(c.name, append(xdm.Sequence(nil), seq[s-1:e]...)) } diff --git a/xquery/window_alloc_test.go b/xquery/window_alloc_test.go new file mode 100644 index 00000000..9f050cb5 --- /dev/null +++ b/xquery/window_alloc_test.go @@ -0,0 +1,30 @@ +package xquery_test + +import ( + "testing" + + "github.com/knroy/go-xml/v2/xpath" + "github.com/knroy/go-xml/v2/xquery" +) + +// TestWindowBindAllocs pins the cost of binding a window's boundary +// variables: they are added to the tuple in one pass, so 2000 sliding windows +// of 10 allocate a bounded number of objects. +func TestWindowBindAllocs(t *testing.T) { + q, err := xquery.Compile(`count(for sliding window $w in 1 to 2009 `+ + `start $s at $i previous $p next $n when true() `+ + `end at $e when $e - $i eq 9 return $w)`, xquery.Options{}) + if err != nil { + t.Fatal(err) + } + ctx := xpath.NewContext(nil, xpath.Builtins()) + allocs := testing.AllocsPerRun(3, func() { + if _, err := q.Eval(ctx); err != nil { + t.Fatal(err) + } + }) + t.Logf("allocs=%v", allocs) + if allocs > 450000 { + t.Fatalf("window binding allocated %v objects, want <= 450000", allocs) + } +} From 857790679ce0dab19ee588f438f3bd34aafe7226 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:22:43 -0400 Subject: [PATCH 66/98] xpath: Env.MaxDigits refuses over-long numeric casts and arithmetic results --- xpath/builtins.go | 5 +++- xpath/context.go | 61 +++++++++++++++++++++++++++++++++++++++++ xpath/maxdigits_test.go | 40 +++++++++++++++++++++++++++ xpath/operators.go | 6 ++++ xpath/typeexpr.go | 3 ++ 5 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 xpath/maxdigits_test.go diff --git a/xpath/builtins.go b/xpath/builtins.go index 0992399d..a4713186 100644 --- a/xpath/builtins.go +++ b/xpath/builtins.go @@ -197,7 +197,7 @@ func registerConstructors(l *Library) { for name, code := range types { target := code - l.register(xdm.NSXS, name, 1, func(_ *Context, args []xdm.Sequence) (xdm.Sequence, error) { + l.register(xdm.NSXS, name, 1, func(ctx *Context, args []xdm.Sequence) (xdm.Sequence, error) { atoms := xdm.Atomize(args[0]) // A constructor applied to the empty sequence yields the empty // sequence rather than raising, so "xs:date($optional)" is safe. @@ -208,6 +208,9 @@ func registerConstructors(l *Library) { if err != nil { return nil, err } + if err := ctx.checkCastDigits(it.(*xdm.Atomic), target); err != nil { + return nil, err + } out, err := CastAtomic(it.(*xdm.Atomic), target) if err != nil { return nil, err diff --git a/xpath/context.go b/xpath/context.go index e3d0d567..e7cc4899 100644 --- a/xpath/context.go +++ b/xpath/context.go @@ -256,6 +256,13 @@ type Env struct { // AdoptBudget copies it along with the shared counter, so a nested // evaluation cannot raise the bound it is charged against. MaxItems int + // MaxDigits bounds the decimal digits of an integer or decimal that a + // cast reads from text or an arithmetic operator produces. Zero means + // MaxDigits' package default, 10^6; a negative value means no bound. A + // cast over it is FOCA0003 and a result over it FOAR0002, both carrying + // xdm.ErrResourceLimit: parsing and multiplying are quadratic in the + // digit count, so an unbounded value is a time and memory attack. + MaxDigits int // items counts the items materialised into intermediate sequences during // this evaluation, bounding memory the way Depth bounds stack. // @@ -1378,6 +1385,60 @@ func (c *Context) ItemsRefused(total int64) error { return nil } +// MaxDigits is the digit bound in force where Env.MaxDigits is zero. +const MaxDigits = 1000000 + +// digitLimit is the digit bound in force, or -1 for none. +func (c *Context) digitLimit() int { + switch { + case c == nil || c.ev().MaxDigits == 0: + return MaxDigits + case c.ev().MaxDigits < 0: + return -1 + } + return c.ev().MaxDigits +} + +// checkCastDigits refuses a string cast to xs:integer or xs:decimal whose text +// is longer than the digit bound, before it is parsed. +func (c *Context) checkCastDigits(a *xdm.Atomic, target xdm.TypeCode) error { + if target != xdm.TypeInteger && target != xdm.TypeDecimal || + a.Type != xdm.TypeString && a.Type != xdm.TypeUntypedAtomic { + return nil + } + if lim := c.digitLimit(); lim >= 0 && len(a.Str()) > lim { + return &xdm.Error{ + Code: "FOCA0003", + Message: fmt.Sprintf("a number of more than %d digits: %s", + lim, xdm.ErrResourceLimit), + Err: xdm.ErrResourceLimit, + } + } + return nil +} + +// checkResultDigits refuses an exact arithmetic result of more than the digit +// bound. A value held in an int64 has no big.Rat and is always small enough. +func (c *Context) checkResultDigits(a *xdm.Atomic) error { + r := a.Rat() + if r == nil { + return nil + } + lim := c.digitLimit() + if lim < 0 { + return nil + } + if bits := max(r.Num().BitLen(), r.Denom().BitLen()); bits*30103/100000 > lim { + return &xdm.Error{ + Code: "FOAR0002", + Message: fmt.Sprintf("a result of more than %d digits: %s", + lim, xdm.ErrResourceLimit), + Err: xdm.ErrResourceLimit, + } + } + return nil +} + // itemLimit is the item bound in force: Env.MaxItems where the caller set // a positive one, none where it set a negative one, MaxItems otherwise. func (c *Context) itemLimit() int64 { diff --git a/xpath/maxdigits_test.go b/xpath/maxdigits_test.go new file mode 100644 index 00000000..7e0eb289 --- /dev/null +++ b/xpath/maxdigits_test.go @@ -0,0 +1,40 @@ +package xpath + +import ( + "errors" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// Env.MaxDigits refuses a cast from over-long text with FOCA0003 and an +// arithmetic result over the bound with FOAR0002, both as resource limits. +func TestMaxDigits(t *testing.T) { + run := func(q string, digits int) error { + c, err := CompileWith(q, CompileOptions{Version: XPath31}) + if err != nil { + t.Fatal(err) + } + ctx := NewContext(nil, Builtins()).WithEnv(func(e *Env) { e.MaxDigits = digits }) + ctx = ctx.WithVar(xdm.QName{Local: "n"}, xdm.One(xdm.NewString(strings.Repeat("9", 101)))) + _, err = c.Eval(ctx) + return err + } + for _, tc := range []struct{ q, code string }{ + {`xs:integer($n)`, "FOCA0003"}, + {`$n cast as xs:decimal`, "FOCA0003"}, + {`let $x := xs:integer(substring($n, 1, 60)) return $x * $x`, "FOAR0002"}, + } { + err := run(tc.q, 100) + if err == nil || !strings.Contains(err.Error(), tc.code) || !errors.Is(err, xdm.ErrResourceLimit) { + t.Errorf("%s: got %v, want %s as a resource limit", tc.q, err, tc.code) + } + } + if err := run(`xs:integer(substring($n, 1, 100))`, 100); err != nil { + t.Errorf("a number at the bound: %v", err) + } + if err := run(`xs:integer($n)`, -1); err != nil { + t.Errorf("unbounded: %v", err) + } +} diff --git a/xpath/operators.go b/xpath/operators.go index bb7937d7..b42e37ce 100644 --- a/xpath/operators.go +++ b/xpath/operators.go @@ -1034,6 +1034,9 @@ func (e *BinaryOp) evalArithmetic(ctx *Context) (xdm.Sequence, error) { if err != nil { return nil, err } + if err := ctx.checkResultDigits(res); err != nil { + return nil, err + } return xdm.One(res), nil } @@ -1053,6 +1056,9 @@ func (e *BinaryOp) evalArithmetic(ctx *Context) (xdm.Sequence, error) { if err != nil { return nil, err } + if err := ctx.checkResultDigits(res); err != nil { + return nil, err + } return xdm.One(res), nil } diff --git a/xpath/typeexpr.go b/xpath/typeexpr.go index 210b1628..e13e5e0c 100644 --- a/xpath/typeexpr.go +++ b/xpath/typeexpr.go @@ -596,6 +596,9 @@ func (e *CastExpr) Eval(ctx *Context) (xdm.Sequence, error) { src.Type == xdm.TypeUntypedAtomic { src = xdm.NewString(src.String()) } + if err := ctx.checkCastDigits(src, e.Type.AtomicType); err != nil { + return nil, err + } out, err := CastToDerived(src, e.Type.AtomicType, e.Type.FacetName) if err == nil && e.Type.SchemaValueValid != nil { // The built-in cast succeeded, which settles the lexical form. The From 6bab64ac0c0396b4df8eeb541f42f6f5985f358b Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:23:19 -0400 Subject: [PATCH 67/98] xpath: fn:replace estimates its result from the matched text and returns early on no match --- xpath/fn_regex.go | 26 ++++++++++++++++++------- xpath/replace_estimate_test.go | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 xpath/replace_estimate_test.go diff --git a/xpath/fn_regex.go b/xpath/fn_regex.go index 8ef0651a..5c647b25 100644 --- a/xpath/fn_regex.go +++ b/xpath/fn_regex.go @@ -133,19 +133,31 @@ func registerRegexFuncs(l *Library) { if err != nil { return nil, err } + // No match leaves the input as it is: nothing is built and nothing is + // charged. + if !re.MatchString(s) { + return strSeq(s), nil + } // The result is charged before it is built: 500 KB of input with a 4 KB // replacement allocated 12.8 GB before the old after-the-fact charge - // refused it. The estimate is the input plus one replacement per match; - // only what the built string exceeds it by is charged afterwards. - n := 0 - re.ReplaceAllStringFunc(s, func(string) string { n++; return "" }) - est := len(s) + n*len(goRepl) + // refused it. The estimate is an upper bound -- the unmatched text, one + // replacement per match, and for each "$" in the replacement at most + // the matched text again, since a group lies inside its match. What + // the built string comes to is then settled against it, so a bound + // that was generous is handed back rather than kept. + n, matched := 0, 0 + re.ReplaceAllStringFunc(s, func(m string) string { n++; matched += len(m); return "" }) + est := len(s) - matched + n*len(goRepl) + strings.Count(goRepl, "$")*matched if err := ctx.countBytes(est); err != nil { return nil, err } out := re.ReplaceAllString(s, goRepl) - if err := ctx.countBytes(len(out) - est); err != nil { - return nil, err + if len(out) > est { + if err := ctx.countBytes(len(out) - est); err != nil { + return nil, err + } + } else { + ctx.refundBytes(est - len(out)) } return strSeq(out), nil }) diff --git a/xpath/replace_estimate_test.go b/xpath/replace_estimate_test.go new file mode 100644 index 00000000..1a2b6c66 --- /dev/null +++ b/xpath/replace_estimate_test.go @@ -0,0 +1,35 @@ +package xpath + +import ( + "strings" + "testing" +) + +// The pre-charge for fn:replace estimates the result from the matched text, +// not the whole input: a result that is empty, or the input unchanged, must +// not be refused for the length of the input. +func TestReplaceChargeFollowsTheResult(t *testing.T) { + in := strings.Repeat("abcdefgh", 100) // 800 bytes + run := func(pat, repl string) error { + ctx := nearlyExhausted(t, 100, 16) + _, err := callFn(t, ctx, "replace", strSeq(in), strSeq(pat), strSeq(repl)) + return err + } + if err := run("abcdefgh", ""); err != nil { + t.Errorf("an empty result was refused: %v", err) + } + if err := run("zzz", "x"); err != nil { + t.Errorf("an unchanged input was refused: %v", err) + } + // A group reference is bounded by the matched text, which over-estimates + // here (1,200 against a 100-byte result); the difference is handed back, + // so the same call can be made again on the same budget. + ctx := nearlyExhausted(t, 1500, 16) + for i := 0; i < 3; i++ { + if _, err := callFn(t, ctx, "replace", strSeq(in), strSeq("(a)bcdefgh"), strSeq("$1")); err != nil { + t.Fatalf("call %d: the over-estimate was not refunded: %v", i, err) + } + } + _, err := callFn(t, nearlyExhausted(t, 100, 16), "replace", strSeq(in), strSeq("a"), strSeq("xyz")) + wantRefused(t, err, "") +} From 40085f6154e48e2f55e07109e6635afe595d8b45 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:23:41 -0400 Subject: [PATCH 68/98] xpath: int64 fast paths for mod, idiv, unary minus and integer comparison --- xpath/intfastpath_test.go | 61 +++++++++++++++++++++++++++++++++++++++ xpath/operators.go | 21 ++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 xpath/intfastpath_test.go diff --git a/xpath/intfastpath_test.go b/xpath/intfastpath_test.go new file mode 100644 index 00000000..cd685001 --- /dev/null +++ b/xpath/intfastpath_test.go @@ -0,0 +1,61 @@ +package xpath + +import ( + "math" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// mod, idiv, unary minus and integer comparison take an int64 path, with the +// results the exact path gives, including at the int64 edges. +func TestIntegerFastPaths(t *testing.T) { + min, max := int64(math.MinInt64), int64(math.MaxInt64) + for _, tc := range []struct { + a, b int64 + op string + want string + }{ + {7, 3, "mod", "1"}, {-7, 3, "mod", "-1"}, {7, -3, "mod", "1"}, {-7, -3, "mod", "-1"}, + {min, -1, "mod", "0"}, {7, 3, "idiv", "2"}, {-7, 3, "idiv", "-2"}, {7, -3, "idiv", "-2"}, + {min, -1, "idiv", "9223372036854775808"}, {max, 1, "idiv", "9223372036854775807"}, + } { + got, err := arithmetic(xdm.NewInteger(tc.a), xdm.NewInteger(tc.b), tc.op) + if err != nil || got.String() != tc.want || got.Type != xdm.TypeInteger { + t.Errorf("%d %s %d = %v, %v; want %s", tc.a, tc.op, tc.b, got, err, tc.want) + } + } + for _, op := range []string{"mod", "idiv"} { + if _, err := arithmetic(xdm.NewInteger(1), xdm.NewInteger(0), op); err == nil { + t.Errorf("1 %s 0: want FOAR0001", op) + } + } + if got := negate(xdm.NewInteger(min)); got.String() != "9223372036854775808" { + t.Errorf("-MinInt64 = %v", got) + } + if got := negate(xdm.NewInteger(5)); got.String() != "-5" { + t.Errorf("-5 = %v", got) + } + if compareNumeric(xdm.NewInteger(min), xdm.NewInteger(max)) != -1 || + compareNumeric(xdm.NewInteger(3), xdm.NewInteger(3)) != 0 || + compareNumeric(xdm.NewInteger(4), xdm.NewInteger(3)) != 1 { + t.Error("integer comparison") + } + + // The saving is the point: these were 25, 13, 6 and 2 allocations. + a, b := xdm.NewInteger(1000003), xdm.NewInteger(1009) + for _, tc := range []struct { + name string + max float64 + f func() + }{ + {"mod", 4, func() { _, _ = arithmetic(a, b, "mod") }}, + {"idiv", 4, func() { _, _ = arithmetic(a, b, "idiv") }}, + {"negate", 4, func() { negate(a) }}, + {"compare", 0, func() { compareNumeric(a, b) }}, + } { + if n := testing.AllocsPerRun(100, tc.f); n > tc.max { + t.Errorf("%s: %v allocs, want <= %v (NewInteger itself is 3)", tc.name, n, tc.max) + } + } +} diff --git a/xpath/operators.go b/xpath/operators.go index b42e37ce..c42133d4 100644 --- a/xpath/operators.go +++ b/xpath/operators.go @@ -2,6 +2,7 @@ package xpath import ( "bytes" + "cmp" "encoding/base64" "encoding/hex" "fmt" @@ -744,6 +745,12 @@ func isOperandDuration(t xdm.TypeCode) bool { func compareNumeric(a, b *xdm.Atomic) int { common := xdm.NumericPromote(a.Type, b.Type) if common == xdm.TypeInteger || common == xdm.TypeDecimal { + if x, y := a.Rat().Num(), b.Rat().Num(); common == xdm.TypeInteger && + x.IsInt64() && y.IsInt64() { + // Integers compare by numerator; Rat.Cmp multiplies the + // cross terms into fresh big.Ints. + return cmp.Compare(x.Int64(), y.Int64()) + } return a.Rat().Cmp(b.Rat()) } af, bf := a.Float64(), b.Float64() @@ -1153,6 +1160,17 @@ func int64Arithmetic(x, y *big.Rat, op string) (z int64, ok bool) { return z, z/b == a && !(a == -1 && b == math.MinInt64) && !(b == -1 && a == math.MinInt64) } + // mod and idiv truncate toward zero, as Go's % and / do. A zero divisor + // and MinInt64 idiv -1 are left to the exact path, which raises or widens. + if b == 0 { + return 0, false + } + switch op { + case "mod": + return a % b, true + case "idiv": + return a / b, !(a == math.MinInt64 && b == -1) + } return 0, false } @@ -1276,6 +1294,9 @@ func toNumeric(a *xdm.Atomic) (*xdm.Atomic, error) { func negate(a *xdm.Atomic) *xdm.Atomic { switch a.Type { case xdm.TypeInteger: + if n := a.Rat().Num(); n.IsInt64() && n.Int64() != math.MinInt64 { + return xdm.NewInteger(-n.Int64()) + } return xdm.NewIntegerFromRat(new(big.Rat).Neg(a.Rat())) case xdm.TypeDecimal: return xdm.NewDecimal(new(big.Rat).Neg(a.Rat())) From 6ea432fe96a1862dbfc8142b231168f9abd8ad19 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:23:54 -0400 Subject: [PATCH 69/98] xslt: slim sortValue and allocate xsl:sort keys in one array --- xslt/instructions.go | 106 ++++++++++++++++++++++++++++------------- xslt/sortvalue_test.go | 15 ++++++ 2 files changed, 89 insertions(+), 32 deletions(-) create mode 100644 xslt/sortvalue_test.go diff --git a/xslt/instructions.go b/xslt/instructions.go index 165805a4..c4058fe4 100644 --- a/xslt/instructions.go +++ b/xslt/instructions.go @@ -1963,8 +1963,12 @@ func applySorts(rt *runtime, seq xdm.Sequence, sorts []*sortKey) (xdm.Sequence, return seq, nil } + // One backing array for every item's keys, rather than a slice header + // and an allocation per item. + nk := len(sorts) + keyBuf := make([]sortValue, n*nk) for i, it := range seq { - e := entry{item: it, idx: i, keys: make([]sortValue, len(sorts))} + e := entry{item: it, idx: i, keys: keyBuf[i*nk : (i+1)*nk : (i+1)*nk]} // Section 16.6.1: current() is "the item that was the context item at // the point where the expression was invoked from the XSLT // stylesheet". A sort key expression is invoked from xsl:sort with @@ -2030,28 +2034,24 @@ func applySorts(rt *runtime, seq xdm.Sequence, sorts []*sortKey) (xdm.Sequence, } // sortValue is a precomputed sort key. +// +// One is held per item per key for the length of a sort, so the layout matters: +// the fields every key carries come first, packed, and the ones only a +// collation or case-order key needs hang off ext. type sortValue struct { - numeric bool - num float64 - str string - // fold is the case-insensitive form, compared first when case-order is - // set so that "a" and "A" sort adjacently rather than in separate blocks. - fold string // upperFirst selects which of an otherwise-equal pair wins. upperFirst bool caseOrder bool - // strColl is the collation from xsl:sort/@collation, if one was named. - strColl xpath.Collation - // collFoldKey is collKey for the case-folded form of str, present only - // when @lang and @case-order are both given. See where it is built. - collFoldKey []byte - // collKey is the locale-aware sort key for str, precomputed because a - // collator is stateful and cannot be shared across the comparisons that - // sort.Slice runs. Comparing two of these byte slices is equivalent to - // asking the collator to compare the strings, and is safe concurrently. - collKey []byte + numeric bool // empty marks an absent key, which sorts before everything else. empty bool + // implicitTZ is needed to order the date/time types, whose comparison + // depends on the timezone an untimezoned value is taken to be in. + implicitTZ int32 + num float64 + str string + // strColl is the collation from xsl:sort/@collation, if one was named. + strColl xpath.Collation // atom is the atomized sort key value, kept for the XSLT 2.0 default // where the comparison follows the value's own type rather than a // conversion the stylesheet asked for. nil when data-type forced a @@ -2067,9 +2067,51 @@ type sortValue struct { // (1 to 5, 'fred') never reached the branch, because 'fred' arrived with // atom already cleared. error-1030a is exactly that sort. cmpAtom *xdm.Atomic - // implicitTZ is needed to order the date/time types, whose comparison - // depends on the timezone an untimezoned value is taken to be in. - implicitTZ int + // ext is set only for a key with @lang or @case-order. + ext *sortExt +} + +// sortExt holds what only a case-ordered or locale-aware key carries. +type sortExt struct { + // fold is the case-insensitive form, compared first when case-order is + // set so that "a" and "A" sort adjacently rather than in separate blocks. + fold string + // collKey is the locale-aware sort key for str, precomputed because a + // collator is stateful and cannot be shared across the comparisons that + // sort.Slice runs. Comparing two of these byte slices is equivalent to + // asking the collator to compare the strings, and is safe concurrently. + collKey []byte + // collFoldKey is collKey for the case-folded form of str, present only + // when @lang and @case-order are both given. See where it is built. + collFoldKey []byte +} + +func (v *sortValue) setFold(f string) { + if v.ext == nil { + v.ext = &sortExt{} + } + v.ext.fold = f +} + +func (v *sortValue) fold() string { + if v.ext == nil { + return "" + } + return v.ext.fold +} + +func (v *sortValue) collKey() []byte { + if v.ext == nil { + return nil + } + return v.ext.collKey +} + +func (v *sortValue) collFoldKey() []byte { + if v.ext == nil { + return nil + } + return v.ext.collFoldKey } // makeSortValue precomputes one sort key. @@ -2103,7 +2145,7 @@ func makeSortValue(seq xdm.Sequence, s *sortKey, coll xpath.Collation, implicitT return sortValue{numeric: true, num: conv.Float64()}, nil } - v := sortValue{str: text, strColl: coll, implicitTZ: implicitTZ} + v := sortValue{str: text, strColl: coll, implicitTZ: int32(implicitTZ)} // With no data-type, the value keeps its own type and the comparison is // the XPath "lt" operator: a sequence of xs:integer sorts numerically, // dates sort chronologically. Only a single atomic value can be ordered @@ -2134,7 +2176,7 @@ func makeSortValue(seq xdm.Sequence, s *sortKey, coll xpath.Collation, implicitT } } if s.coll != nil { - v.collKey = s.coll.key(text) + v.ext = &sortExt{collKey: s.coll.key(text)} if s.caseOrder == "upper-first" || s.caseOrder == "lower-first" { // case-order is the caseFirst tailoring: it decides which of a // pair that agrees on every other weight comes first. x/text @@ -2143,7 +2185,7 @@ func makeSortValue(seq xdm.Sequence, s *sortKey, coll xpath.Collation, implicitT // the exact ties. The case-folded key is taken through the same // collator so the language's own placement of accented letters // still governs the primary ordering. - v.collFoldKey = s.coll.key(strings.ToLower(text)) + v.ext.collFoldKey = s.coll.key(strings.ToLower(text)) } } // case-order only has an effect on text sorts; without it a plain @@ -2152,10 +2194,10 @@ func makeSortValue(seq xdm.Sequence, s *sortKey, coll xpath.Collation, implicitT switch s.caseOrder { case "upper-first": v.caseOrder, v.upperFirst = true, true - v.fold = strings.ToLower(text) + v.setFold(strings.ToLower(text)) case "lower-first": v.caseOrder, v.upperFirst = true, false - v.fold = strings.ToLower(text) + v.setFold(strings.ToLower(text)) } return v, nil } @@ -2206,7 +2248,7 @@ func compareSortValues(a, b sortValue) int { // A language-sensitive collation replaces codepoint order entirely: it // already places accented and cased letters where that language expects. - if a.collKey != nil && b.collKey != nil { + if a.collKey() != nil && b.collKey() != nil { // @case-order alongside @lang does not fight the collation, it // tailors it: 13.1.3 makes case-order the choice of which of two // values that are "equal apart from case" comes first, which is @@ -2214,11 +2256,11 @@ func compareSortValues(a, b sortValue) int { // setting. So the case-insensitive keys order the pair, and case // only breaks a tie the language's own rules left. if a.caseOrder && b.caseOrder && - a.collFoldKey != nil && b.collFoldKey != nil { - if c := bytes.Compare(a.collFoldKey, b.collFoldKey); c != 0 { + a.collFoldKey() != nil && b.collFoldKey() != nil { + if c := bytes.Compare(a.collFoldKey(), b.collFoldKey()); c != 0 { return c } - if c := bytes.Compare(a.collKey, b.collKey); c == 0 { + if c := bytes.Compare(a.collKey(), b.collKey()); c == 0 { return 0 } c := strings.Compare(a.str, b.str) @@ -2232,7 +2274,7 @@ func compareSortValues(a, b sortValue) int { } return -c } - if c := bytes.Compare(a.collKey, b.collKey); c != 0 { + if c := bytes.Compare(a.collKey(), b.collKey()); c != 0 { return c } // Equal under the collation but not identical: fall through to @@ -2243,7 +2285,7 @@ func compareSortValues(a, b sortValue) int { // With case-order set, values are ordered case-insensitively first and the // case distinction only breaks ties. if a.caseOrder && b.caseOrder { - if c := strings.Compare(a.fold, b.fold); c != 0 { + if c := strings.Compare(a.fold(), b.fold()); c != 0 { return c } c := strings.Compare(a.str, b.str) @@ -2265,7 +2307,7 @@ func compareSortValues(a, b sortValue) int { // when the values genuinely have no ordering and the suite expects the // sort to complete. if a.atom != nil && b.atom != nil { - if c, ok := compareAtoms(a.atom, b.atom, a.implicitTZ); ok { + if c, ok := compareAtoms(a.atom, b.atom, int(a.implicitTZ)); ok { return c } // XTDE1030: "it is a non-recoverable dynamic error if, for any sort diff --git a/xslt/sortvalue_test.go b/xslt/sortvalue_test.go new file mode 100644 index 00000000..74ef6065 --- /dev/null +++ b/xslt/sortvalue_test.go @@ -0,0 +1,15 @@ +package xslt + +import ( + "testing" + "unsafe" +) + +// applySorts holds one sortValue per item per key. It was 152 bytes, with the +// collation and case-fold material inline; that material now hangs off a +// pointer, which only the keys needing it pay for. +func TestSortValueStaysSmall(t *testing.T) { + if sz := unsafe.Sizeof(sortValue{}); sz > 96 { + t.Fatalf("sortValue is %d bytes, want at most 96", sz) + } +} From c73a06b6f607882ffe5b2b8b4887f3d425945ed6 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:24:10 -0400 Subject: [PATCH 70/98] xpath: charge the lines of unparsed-text-lines and the members of parse-json and json-doc --- xpath/fn_json.go | 7 ++++++- xpath/fn_misc.go | 15 ++++++++++----- xpath/io_budget_test.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 xpath/io_budget_test.go diff --git a/xpath/fn_json.go b/xpath/fn_json.go index 319f3c54..ba1f6db0 100644 --- a/xpath/fn_json.go +++ b/xpath/fn_json.go @@ -882,6 +882,8 @@ type jsonBuilder struct { // to; finishing one pops it and hands the finished item to its parent. stack []*jsonFrame opts jsonOptions + // ctx is charged one item for every value the parse produces. + ctx *Context // result is the top-level value, once one has been produced. result xdm.Sequence done bool @@ -901,6 +903,9 @@ type jsonFrame struct { } func (b *jsonBuilder) emit(v xdm.Sequence) error { + if err := b.ctx.countItems(1); err != nil { + return err + } if len(b.stack) == 0 { if b.done { return errFOJS0001("more than one value at the top level") @@ -1023,7 +1028,7 @@ func jsonDocReadError(err error) error { // parseJSONText is the shared body of fn:parse-json and fn:json-doc. func parseJSONText(ctx *Context, text string, opts jsonOptions) (xdm.Sequence, error) { - b := &jsonBuilder{opts: opts} + b := &jsonBuilder{opts: opts, ctx: ctx} fb := jsonFallback(ctx, opts) if err := scanJSON(text, opts, b, fb); err != nil { return nil, err diff --git a/xpath/fn_misc.go b/xpath/fn_misc.go index 1fecaa0c..26d757fe 100644 --- a/xpath/fn_misc.go +++ b/xpath/fn_misc.go @@ -198,7 +198,7 @@ func registerUnparsedText(l *Library, since Version) { if err != nil { return nil, err } - return xdm.Sequence(splitTextLines(text)), nil + return splitTextLines(ctx, text) }) } @@ -298,21 +298,26 @@ func readText(ctx *Context, s, enc string) (string, error) { // The spec defines the result as the sequence obtained by splitting on \n, // \r\n or \r, with a trailing line ending producing no final empty string — // "a\n" is one line, not two. -func splitTextLines(text string) []xdm.Item { +func splitTextLines(ctx *Context, text string) (xdm.Sequence, error) { if text == "" { - return nil + return nil, nil } // Normalise the two-character ending first so the split is over single // characters, then treat a lone \r as an ending too. text = strings.ReplaceAll(text, "\r\n", "\n") text = strings.ReplaceAll(text, "\r", "\n") text = strings.TrimSuffix(text, "\n") + // Charge the lines before the strings.Split slice and the items are built: + // a 67 MB file of newlines is 67M items. + out, err := makeSequence(ctx, strings.Count(text, "\n")+1) + if err != nil { + return nil, err + } parts := strings.Split(text, "\n") - out := make([]xdm.Item, 0, len(parts)) for _, p := range parts { out = append(out, xdm.NewString(p)) } - return out + return out, nil } // unparsedTextArgs pulls the href and the optional encoding out of a call on diff --git a/xpath/io_budget_test.go b/xpath/io_budget_test.go new file mode 100644 index 00000000..a08813c2 --- /dev/null +++ b/xpath/io_budget_test.go @@ -0,0 +1,31 @@ +package xpath + +import ( + "strings" + "testing" +) + +// fn:parse-json, fn:json-doc and fn:unparsed-text-lines build one item per +// member or line, and must charge them to the item budget as they go. +func TestJSONMembersAreCharged(t *testing.T) { + ctx := nearlyExhausted(t, 1<<20, 5) + _, err := callFn(t, ctx, "parse-json", strSeq("[1,2,3,4,5,6,7,8,9]")) + wantRefused(t, err, "") + ctx = nearlyExhausted(t, 1<<20, 5) + _, err = callFn(t, ctx, "parse-json", strSeq(`{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6}`)) + wantRefused(t, err, "") + ctx = nearlyExhausted(t, 1<<20, 50) + if _, err = callFn(t, ctx, "parse-json", strSeq(`{"a":[1,2],"b":2}`)); err != nil { + t.Fatalf("a small document was refused: %v", err) + } +} + +func TestTextLinesAreCharged(t *testing.T) { + text := strings.Repeat("x\n", 9) + _, err := splitTextLines(nearlyExhausted(t, 1<<20, 5), text) + wantRefused(t, err, "") + got, err := splitTextLines(nearlyExhausted(t, 1<<20, 50), "a\r\nb\rc\n") + if err != nil || len(got) != 3 { + t.Fatalf("got %v, %v; want 3 lines", got, err) + } +} From 98811f6f2a9765e3c6fb097619525b488d1f02a2 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:24:41 -0400 Subject: [PATCH 71/98] xpath: build parse-json objects with xdm.MapBuilder --- xpath/fn_json.go | 8 +++----- xpath/json_map_alloc_test.go | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 xpath/json_map_alloc_test.go diff --git a/xpath/fn_json.go b/xpath/fn_json.go index ba1f6db0..a39ca1fb 100644 --- a/xpath/fn_json.go +++ b/xpath/fn_json.go @@ -948,15 +948,13 @@ func (b *jsonBuilder) key(k string) error { func (b *jsonBuilder) endMap() error { f := b.stack[len(b.stack)-1] b.stack = b.stack[:len(b.stack)-1] - m := xdm.NewMap() + mb := xdm.NewMapBuilder() for i, k := range f.keys { - next, err := m.Put(xdm.NewString(k), f.values[i]) - if err != nil { + if err := mb.Set(xdm.NewString(k), f.values[i]); err != nil { return err } - m = next } - return b.emit(xdm.One(m)) + return b.emit(xdm.One(mb.Build())) } func (b *jsonBuilder) startArray() error { diff --git a/xpath/json_map_alloc_test.go b/xpath/json_map_alloc_test.go new file mode 100644 index 00000000..b529b8de --- /dev/null +++ b/xpath/json_map_alloc_test.go @@ -0,0 +1,37 @@ +package xpath + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// endMap builds the parsed object through xdm.MapBuilder, which writes the +// trie in place; Put allocated a path per key (2.6 KB a key at 1M keys). +func TestJSONObjectBuildAllocation(t *testing.T) { + const keys = 20000 + var sb strings.Builder + sb.WriteByte('{') + for i := 0; i < keys; i++ { + if i > 0 { + sb.WriteByte(',') + } + fmt.Fprintf(&sb, `"k%d":1`, i) + } + sb.WriteByte('}') + text := sb.String() + var a, b runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&a) + seq, err := parseJSONText(NewContext(nil, Builtins()), text, jsonOptions{}) + runtime.ReadMemStats(&b) + if err != nil || len(seq) != 1 { + t.Fatalf("parse: %v %v", seq, err) + } + perKey := (b.TotalAlloc - a.TotalAlloc) / keys + t.Logf("%d bytes allocated per key", perKey) + if perKey > 1000 { + t.Errorf("%d bytes allocated per key; want at most %d", perKey, 1000) + } +} From 951f0d962acc8b3bd6919608eb2ca28a67ad8c81 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:25:03 -0400 Subject: [PATCH 72/98] xslt: check XTDE1490 against sets instead of scanning earlier result documents --- xslt/resultdoc.go | 19 +++++----- xslt/resultdocscale_test.go | 71 +++++++++++++++++++++++++++++++++++++ xslt/runtime.go | 6 ++++ 3 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 xslt/resultdocscale_test.go diff --git a/xslt/resultdoc.go b/xslt/resultdoc.go index 009f5995..744e4d12 100644 --- a/xslt/resultdoc.go +++ b/xslt/resultdoc.go @@ -261,13 +261,15 @@ func (i *resultDocumentInstr) Execute(rt *runtime, out *outputBuilder) error { // output URI is known: resolution then yields "" for every document, and // the resolved form would neither catch a real collision nor be safe to // treat as one. - for _, prev := range *rt.secondary { - if prev.Href == href || - (resolvedHref != "" && prev.BaseURI == resolvedHref) { - return fmt.Errorf( - "XTDE1490: xsl:result-document: href %q was already produced by an earlier "+ - "result document", href) - } + // + // Both identities are looked up in a set: the raw href in resultHrefs and + // the resolved URI in writtenDocs, which records every resolved URI a + // result document has written. + if (*rt.resultHrefs)[href] || + (resolvedHref != "" && (*rt.writtenDocs)[resolvedHref]) { + return fmt.Errorf( + "XTDE1490: xsl:result-document: href %q was already produced by an earlier "+ + "result document", href) } settings := sepSettings @@ -330,7 +332,8 @@ func (i *resultDocumentInstr) Execute(rt *runtime, out *outputBuilder) error { // the read-then-write half was detected; see checkWrittenThenRead. // Keyed on the resolved form for the same reason checkReadThenWrite is: // the spec's test is on the absolute URI. - if rt.writtenDocs != nil && resolvedHref != "" { + (*rt.resultHrefs)[href] = true + if resolvedHref != "" { (*rt.writtenDocs)[resolvedHref] = true } return nil diff --git a/xslt/resultdocscale_test.go b/xslt/resultdocscale_test.go new file mode 100644 index 00000000..95f91813 --- /dev/null +++ b/xslt/resultdocscale_test.go @@ -0,0 +1,71 @@ +package xslt + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/knroy/go-xml/v2/xdm" +) + +// The XTDE1490 check scanned every earlier result document, so n documents +// cost n^2/2 comparisons (40k documents took 9.9 s). It is now a set lookup. +func TestResultDocumentCollisionCheckIsLinear(t *testing.T) { + const sheet = ` + + + + + + + + ` + st, err := xdm.ParseString(sheet, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + c, err := Compile(st.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + src, _ := xdm.ParseString("", xdm.ParseOptions{}) + run := func(n int) time.Duration { + start := time.Now() + res, err := c.Transform(context.Background(), src.Root, TransformOptions{ + Params: map[string]xdm.Sequence{"n": {xdm.NewInteger(int64(n))}}}) + if err != nil { + t.Fatal(err) + } + if len(res.Secondary) != n { + t.Fatalf("n=%d: %d secondary results", n, len(res.Secondary)) + } + return time.Since(start) + } + best := func(n int) time.Duration { + a, b := run(n), run(n) + if b < a { + return b + } + return a + } + small, large := best(3000), best(12000) + // 4x the documents: about 4x the time when linear, about 16x when not. + if large > 9*small { + t.Fatalf("12000 documents took %v against %v for 3000", large, small) + } + + // A repeated href is still XTDE1490. + dup := strings.Replace(sheet, ``, ``, 1) + st2, _ := xdm.ParseString(dup, xdm.ParseOptions{}) + c2, err := Compile(st2.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + _, err = c2.Transform(context.Background(), src.Root, TransformOptions{ + Params: map[string]xdm.Sequence{"n": {xdm.NewInteger(3)}}}) + if err == nil || !strings.Contains(err.Error(), "XTDE1490") { + t.Fatalf("duplicate href: err = %v, want XTDE1490", err) + } +} diff --git a/xslt/runtime.go b/xslt/runtime.go index f0c01b89..a95b5347 100644 --- a/xslt/runtime.go +++ b/xslt/runtime.go @@ -246,6 +246,11 @@ type transformState struct { // appending to a copy that the caller never sees. secondary *[]SecondaryResult + // resultHrefs is the set of @href values, as written, that + // xsl:result-document has produced, so that XTDE1490 is a lookup rather + // than a scan of secondary. A pointer for the same reason secondary is. + resultHrefs *map[string]bool + // baseURIUsed records that an xsl:result-document claimed the base output // URI — the one an absent or empty @href names. The principal result tree // has that URI too, so a stylesheet that writes to both is producing two @@ -764,6 +769,7 @@ func newRuntime(s *Stylesheet, ctx context.Context, root *xdm.Node, opts Transfo messages: new([]string), warnings: new([]string), secondary: new([]SecondaryResult), + resultHrefs: &map[string]bool{}, baseURIUsed: new(bool), baseOutputURI: opts.BaseOutputURI, opts: &opts, From 8b97c7cf2c73482e2ac6ab39db399dfe220fd2f3 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:25:12 -0400 Subject: [PATCH 73/98] xpath: pass value depth down deep-equal and map:find instead of growing Context --- xpath/context.go | 15 ++++++--------- xpath/fn_31.go | 20 +++++++++----------- xpath/fn_map.go | 13 ++++++------- xpath/fn_misc.go | 11 ++++++++--- 4 files changed, 29 insertions(+), 30 deletions(-) diff --git a/xpath/context.go b/xpath/context.go index e7cc4899..d3518cd9 100644 --- a/xpath/context.go +++ b/xpath/context.go @@ -62,9 +62,6 @@ type Context struct { // Depth guards against unbounded recursion in user-defined functions and // named templates, which the spec does not bound. Depth int32 - // valueDepth counts the nesting deep-equal and map:find have descended - // into, apart from Depth so that function recursion does not shrink it. - valueDepth int32 // heldItems suppresses Compiled.Eval's per-expression reset of items, // because a host language is measuring a larger evaluation against the // same counter. Set by HoldItemBudget, and copied along with the rest of @@ -1332,15 +1329,15 @@ func (c *Context) checkDepth() error { return nil } -// enterValue is checkDepth for the nesting of a map or array value, bounded -// at maxJSONDepth like parse-json and counted apart from function recursion. -// The caller decrements valueDepth when it returns. -func (c *Context) enterValue() error { - if c.valueDepth >= maxJSONDepth { +// checkValueDepth bounds the nesting of a map or array value that deep-equal +// or map:find descends into at maxJSONDepth, like parse-json. The depth is +// passed down by the walkers rather than kept in the Context, whose size is +// pinned, and so is counted apart from function recursion. +func checkValueDepth(depth int) error { + if depth >= maxJSONDepth { return fmt.Errorf("XPDY0001: value nests more than %d levels: %w", maxJSONDepth, xdm.ErrResourceLimit) } - c.valueDepth++ return nil } diff --git a/xpath/fn_31.go b/xpath/fn_31.go index 47e9ea9a..5122e73b 100644 --- a/xpath/fn_31.go +++ b/xpath/fn_31.go @@ -1179,7 +1179,7 @@ func permuteWith(state uint64, seq xdm.Sequence) xdm.Sequence { // A map or an array compared against anything else is simply unequal rather // than an error, which is the same rule the atomic branch follows for values // of incomparable types. -func deepEqualMapArray(ctx *Context, x, y xdm.Item) (eq, handled bool, err error) { +func deepEqualMapArray(ctx *Context, x, y xdm.Item, depth int) (eq, handled bool, err error) { xm, xIsMap := x.(*xdm.MapItem) ym, yIsMap := y.(*xdm.MapItem) xa, xIsArr := x.(*xdm.ArrayItem) @@ -1190,10 +1190,10 @@ func deepEqualMapArray(ctx *Context, x, y xdm.Item) (eq, handled bool, err error } switch { case xIsMap && yIsMap: - e, err := deepEqualMaps(ctx, xm, ym) + e, err := deepEqualMaps(ctx, xm, ym, depth) return e, true, err case xIsArr && yIsArr: - e, err := deepEqualArrays(ctx, xa, ya) + e, err := deepEqualArrays(ctx, xa, ya, depth) return e, true, err } // One of the two is a map or an array and the other is not, so they @@ -1202,12 +1202,11 @@ func deepEqualMapArray(ctx *Context, x, y xdm.Item) (eq, handled bool, err error return false, true, nil } -func deepEqualArrays(ctx *Context, a, b *xdm.ArrayItem) (bool, error) { +func deepEqualArrays(ctx *Context, a, b *xdm.ArrayItem, depth int) (bool, error) { // Nested values recurse here; bound them like any other recursion. - if err := ctx.enterValue(); err != nil { + if err := checkValueDepth(depth); err != nil { return false, err } - defer func() { ctx.valueDepth-- }() if a.Len() != b.Len() { return false, nil } @@ -1216,7 +1215,7 @@ func deepEqualArrays(ctx *Context, a, b *xdm.ArrayItem) (bool, error) { // Members are sequences, so this is the sequence comparison rather // than the item one: an array of one two-item member is not equal to // an array of two one-item members. - eq, err := deepEqual(ctx, am[i], bm[i]) + eq, err := deepEqualAt(ctx, am[i], bm[i], depth+1) if err != nil || !eq { return false, err } @@ -1224,12 +1223,11 @@ func deepEqualArrays(ctx *Context, a, b *xdm.ArrayItem) (bool, error) { return true, nil } -func deepEqualMaps(ctx *Context, a, b *xdm.MapItem) (bool, error) { +func deepEqualMaps(ctx *Context, a, b *xdm.MapItem, depth int) (bool, error) { // Nested values recurse here; bound them like any other recursion. - if err := ctx.enterValue(); err != nil { + if err := checkValueDepth(depth); err != nil { return false, err } - defer func() { ctx.valueDepth-- }() if a.Len() != b.Len() { return false, nil } @@ -1252,7 +1250,7 @@ func deepEqualMaps(ctx *Context, a, b *xdm.MapItem) (bool, error) { equal = false return nil } - eq, err := deepEqual(ctx, value, other) + eq, err := deepEqualAt(ctx, value, other, depth+1) if err != nil { return err } diff --git a/xpath/fn_map.go b/xpath/fn_map.go index 5401f417..72df80f4 100644 --- a/xpath/fn_map.go +++ b/xpath/fn_map.go @@ -188,13 +188,12 @@ func registerMapFuncs(l *Library) { return nil, err } var found []xdm.Sequence - var walk func(seq xdm.Sequence) error - walk = func(seq xdm.Sequence) error { + var walk func(seq xdm.Sequence, depth int) error + walk = func(seq xdm.Sequence, depth int) error { // Recursion on the value's own nesting, so bounded like any other. - if err := ctx.enterValue(); err != nil { + if err := checkValueDepth(depth); err != nil { return err } - defer func() { ctx.valueDepth-- }() for _, it := range seq { switch v := it.(type) { case *xdm.MapItem: @@ -208,13 +207,13 @@ func registerMapFuncs(l *Library) { found = append(found, val) } if err := v.Entries(func(_ *xdm.Atomic, val xdm.Sequence) error { - return walk(val) + return walk(val, depth+1) }); err != nil { return err } case *xdm.ArrayItem: for _, mem := range v.Members() { - if err := walk(mem); err != nil { + if err := walk(mem, depth+1); err != nil { return err } } @@ -222,7 +221,7 @@ func registerMapFuncs(l *Library) { } return nil } - if err := walk(args[0]); err != nil { + if err := walk(args[0], 0); err != nil { return nil, err } return xdm.One(xdm.NewArray(found...)), nil diff --git a/xpath/fn_misc.go b/xpath/fn_misc.go index 1fecaa0c..e67517c5 100644 --- a/xpath/fn_misc.go +++ b/xpath/fn_misc.go @@ -493,11 +493,16 @@ func lookupByID(ctx *Context, args []xdm.Sequence, wantID bool) (xdm.Sequence, e // instructions are ignored when comparing element content, which is why this // cannot simply compare serialised forms. func deepEqual(ctx *Context, a, b xdm.Sequence) (bool, error) { + return deepEqualAt(ctx, a, b, 0) +} + +// deepEqualAt is deepEqual for values nested depth maps or arrays down. +func deepEqualAt(ctx *Context, a, b xdm.Sequence, depth int) (bool, error) { if len(a) != len(b) { return false, nil } for i := range a { - eq, err := deepEqualItem(ctx, a[i], b[i]) + eq, err := deepEqualItem(ctx, a[i], b[i], depth) if err != nil || !eq { return false, err } @@ -505,7 +510,7 @@ func deepEqual(ctx *Context, a, b xdm.Sequence) (bool, error) { return true, nil } -func deepEqualItem(ctx *Context, x, y xdm.Item) (bool, error) { +func deepEqualItem(ctx *Context, x, y xdm.Item, depth int) (bool, error) { // An Opaque is engine-internal state, not a value the data model defines // equality over. A stylesheet naming the internal namespace can put one // here, so it is compared by identity rather than asserted to be atomic. @@ -531,7 +536,7 @@ func deepEqualItem(ctx *Context, x, y xdm.Item) (bool, error) { // before the node and atomic branches because neither is either one: the // fallback below compares unrecognised items by identity, so two equal // arrays built separately were answering false. - if eq, handled, err := deepEqualMapArray(ctx, x, y); handled { + if eq, handled, err := deepEqualMapArray(ctx, x, y, depth); handled { return eq, err } From f650a196b04d0632ddbae37baa3c72f01326dbca Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:25:22 -0400 Subject: [PATCH 74/98] xslt: skip the focus copy for built-in rules in a mode with no rules; size namespace decls to need --- xdmbuild/builder.go | 2 +- xslt/apply.go | 24 ++++++++++++++++++------ xslt/builtinfocus_test.go | 38 ++++++++++++++++++++++++++++++++++++++ xslt/ruleindex.go | 8 +++++++- 4 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 xslt/builtinfocus_test.go diff --git a/xdmbuild/builder.go b/xdmbuild/builder.go index b3defda2..3faf4c8e 100644 --- a/xdmbuild/builder.go +++ b/xdmbuild/builder.go @@ -777,7 +777,7 @@ func (b *Builder) setDecl(prefix, uri string) { } } if b.declared == nil { - b.declared = make([]decl, 0, 8) + b.declared = make([]decl, 0, 3) } b.declared = append(b.declared, decl{Prefix: prefix, URI: uri}) b.declShared = false // a lent list has no spare capacity, so append copied it diff --git a/xslt/apply.go b/xslt/apply.go index 4c5964ed..74eb46a1 100644 --- a/xslt/apply.go +++ b/xslt/apply.go @@ -415,7 +415,7 @@ func applyBuiltInRule(rt *runtime, node *xdm.Node, mode string, size, idx := node.NumChildren(), 0 for ch := range node.Children() { idx++ - sub := rt.withCurrent(ch, idx, size) + sub := rt.builtInFocus(mode, ch, idx, size) if err := applyToNode(sub, ch, mode, params, tunnels, out); err != nil { return err } @@ -1140,7 +1140,7 @@ func builtInDescend(rt *runtime, node *xdm.Node, mode string, size, idx := node.NumChildren(), 0 for ch := range node.Children() { idx++ - sub := rt.withCurrent(ch, idx, size) + sub := rt.builtInFocus(mode, ch, idx, size) if err := applyToNode(sub, ch, mode, params, tunnels, out); err != nil { return err } @@ -1179,7 +1179,7 @@ func builtInSkipDescend(rt *runtime, node *xdm.Node, mode string, // the identity rule. for idx := range node.NumAttrs() { a := node.AttrAt(idx) - an := rt.withCurrent(a, idx+1, node.NumAttrs()) + an := rt.builtInFocus(mode, a, idx+1, node.NumAttrs()) if err := applyToNode(an, a, mode, params, tunnels, out); err != nil { return err } @@ -1187,7 +1187,7 @@ func builtInSkipDescend(rt *runtime, node *xdm.Node, mode string, size, idx := node.NumChildren(), 0 for ch := range node.Children() { idx++ - cn := rt.withCurrent(ch, idx, size) + cn := rt.builtInFocus(mode, ch, idx, size) if err := applyToNode(cn, ch, mode, params, tunnels, out); err != nil { return err } @@ -1195,6 +1195,18 @@ func builtInSkipDescend(rt *runtime, node *xdm.Node, mode string, return nil } +// builtInFocus is rt.withCurrent for a child the built-in rules are about to +// visit. When the mode has no template rules nothing reads the focus -- no +// pattern is matched and no body runs -- so the copy of the runtime and of +// its context is skipped. That copy was most of the garbage of an identity +// shallow-copy. +func (rt *runtime) builtInFocus(mode string, item xdm.Item, pos, size int) *runtime { + if mr := rt.sheet.rules[mode]; mr != nil && mr.empty { + return rt + } + return rt.withCurrent(item, pos, size) +} + // builtInShallowCopy is the on-no-match="shallow-copy" rule: copy the node // itself without its content, then process its children into the copy. // @@ -1250,7 +1262,7 @@ func builtInShallowCopy(rt *runtime, node *xdm.Node, mode string, } defer rt.ascend() for a := range node.Attrs() { - an := rt.withCurrent(a, 1, node.NumAttrs()) + an := rt.builtInFocus(mode, a, 1, node.NumAttrs()) if err := applyToNode(an, a, mode, params, tunnels, sub); err != nil { return err } @@ -1258,7 +1270,7 @@ func builtInShallowCopy(rt *runtime, node *xdm.Node, mode string, size, idx := node.NumChildren(), 0 for ch := range node.Children() { idx++ - cn := rt.withCurrent(ch, idx, size) + cn := rt.builtInFocus(mode, ch, idx, size) if err := applyToNode(cn, ch, mode, params, tunnels, sub); err != nil { return err } diff --git a/xslt/builtinfocus_test.go b/xslt/builtinfocus_test.go new file mode 100644 index 00000000..c9cfe363 --- /dev/null +++ b/xslt/builtinfocus_test.go @@ -0,0 +1,38 @@ +package xslt + +import ( + "context" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// A mode with no template rules reads no focus, so the built-in shallow-copy +// rule must not copy the runtime and its XPath context for every node it +// visits. Before, 1,000 t cost about 10,000 allocations (10 per +// element); the budget below is 6 per element. +func TestBuiltInShallowCopyWithoutRulesDoesNotCopyFocus(t *testing.T) { + const n = 1000 + s := compileString(t, ` + `) + src := "" + strings.Repeat(`t`, n) + "" + doc, err := xdm.ParseString(src, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var res *Result + allocs := testing.AllocsPerRun(3, func() { + res, err = s.Transform(context.Background(), doc.Root, TransformOptions{}) + }) + if err != nil { + t.Fatal(err) + } + if got := len(res.Nodes); got != 1 { + t.Fatalf("result has %d items, want 1", got) + } + if per := allocs / n; per > 6 { + t.Errorf("%.1f allocations per element, want <= 6", per) + } +} diff --git a/xslt/ruleindex.go b/xslt/ruleindex.go index 12da4e73..55ea9dae 100644 --- a/xslt/ruleindex.go +++ b/xslt/ruleindex.go @@ -21,6 +21,8 @@ import ( type ruleIndex map[string]*modeRules type modeRules struct { + // empty records that no template rule is in the mode at all. + empty bool byKind [xdm.KindNamespace + 1][]int byName map[ruleName][]int } @@ -43,6 +45,9 @@ func (s *Stylesheet) buildRuleIndex() { } } } + for m := range s.modeNoMatch { + modes[m] = true + } type keys struct { kinds [xdm.KindNamespace + 1]bool names []ruleName @@ -53,11 +58,12 @@ func (s *Stylesheet) buildRuleIndex() { } idx := ruleIndex{} for mode := range modes { - mr := &modeRules{byName: map[ruleName][]int{}} + mr := &modeRules{byName: map[ruleName][]int{}, empty: true} for i, t := range s.templates { if !t.matchesMode(mode) { continue } + mr.empty = false kinds, names := tk[i].kinds, tk[i].names for k := range mr.byKind { if kinds[k] { From 6317d49ec456feadf21b4beda89a63862ec75e64 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:25:23 -0400 Subject: [PATCH 75/98] xpath: fn:serialize stops at the first refused write --- xpath/fn_serialize.go | 6 ++++++ xpath/serialize_sep_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 xpath/serialize_sep_test.go diff --git a/xpath/fn_serialize.go b/xpath/fn_serialize.go index 87a19f9a..bde6d6a9 100644 --- a/xpath/fn_serialize.go +++ b/xpath/fn_serialize.go @@ -115,6 +115,12 @@ func registerSerialize(l *Library) { if err := serializeItem(sb, it, opts); err != nil { return nil, err } + // A refused sink drops every later write, but the loop would + // still walk the rest of the items and their separators. The + // latched field is read directly: err() settles the reserve. + if sb.e != nil { + return nil, sb.err() + } prevAtomic = atomic } if err := sb.err(); err != nil { diff --git a/xpath/serialize_sep_test.go b/xpath/serialize_sep_test.go new file mode 100644 index 00000000..2054d8ee --- /dev/null +++ b/xpath/serialize_sep_test.go @@ -0,0 +1,32 @@ +package xpath + +import ( + "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// Once the sink has refused a write the rest of the sequence is wasted work, +// so fn:serialize must stop at the first refusal instead of walking every item. +func TestSerializeStopsAtTheFirstRefusal(t *testing.T) { + const items = 100000 + seq := make(xdm.Sequence, items) + for i := range seq { + seq[i] = xdm.NewInteger(int64(i)) + } + opts := xdm.NewMapBuilder() + if err := opts.Set(xdm.NewString("item-separator"), xdm.One(xdm.NewString(strings.Repeat("s", 1<<10)))); err != nil { + t.Fatal(err) + } + ctx := nearlyExhausted(t, 8<<10, 16) + var a, b runtime.MemStats + runtime.ReadMemStats(&a) + _, err := callFn(t, ctx, "serialize", seq, xdm.One(opts.Build())) + runtime.ReadMemStats(&b) + wantRefused(t, err, "") + if n := b.Mallocs - a.Mallocs; n > items/10 { + t.Errorf("%d allocations after the refusal; the item loop kept going", n) + } +} From 30c7c0744070cca31bf656656f3023268f72ead4 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:26:32 -0400 Subject: [PATCH 76/98] xdm: allocate range integers a chunk at a time --- xdm/atomic.go | 41 +++++++++++++++++++++++++++++++++++++++++ xdm/integers_test.go | 33 +++++++++++++++++++++++++++++++++ xpath/operators.go | 7 +------ 3 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 xdm/integers_test.go diff --git a/xdm/atomic.go b/xdm/atomic.go index f062320e..4041874d 100644 --- a/xdm/atomic.go +++ b/xdm/atomic.go @@ -669,3 +669,44 @@ func NewGregorian(dt *DateTime, t TypeCode) *Atomic { func NewDuration(d *Duration, t TypeCode) *Atomic { return &Atomic{Type: t, ext: &atomicExt{dur: d}, str: d.Lexical(t)} } + +// integerChunk is how many xs:integers AppendIntegers allocates at once. +const integerChunk = 128 + +// AppendIntegers appends the count integers lo, lo+1, ... to out. It allocates +// the Atomics, rationals and numerator words a chunk at a time rather than +// three allocations per value, which is what NewInteger costs; the values +// are indistinguishable from NewInteger's. The caller must ensure lo+count-1 +// does not overflow int64. +func AppendIntegers(out Sequence, lo int64, count int) Sequence { + var ( + atoms []Atomic + rats []big.Rat + words []big.Word + ) + for i := 0; i < count; i++ { + v := lo + int64(i) + if v >= smallIntMin && v < smallIntMin+int64(len(smallInts)) { + out = append(out, NewInteger(v)) + continue + } + if len(atoms) == 0 { + n := min(integerChunk, count-i) + atoms, rats, words = make([]Atomic, n), make([]big.Rat, n), make([]big.Word, n) + } + a, r := &atoms[0], &rats[0] + if v < 0 { + words[0] = big.Word(-uint64(v)) + } else { + words[0] = big.Word(v) + } + r.Num().SetBits(words[:1:1]) + if v < 0 { + r.Num().Neg(r.Num()) + } + *a = Atomic{Type: TypeInteger, dec: r} + atoms, rats, words = atoms[1:], rats[1:], words[1:] + out = append(out, a) + } + return out +} diff --git a/xdm/integers_test.go b/xdm/integers_test.go new file mode 100644 index 00000000..0708ae04 --- /dev/null +++ b/xdm/integers_test.go @@ -0,0 +1,33 @@ +package xdm + +import ( + "math" + "testing" +) + +// AppendIntegers gives the values NewInteger gives, including across the +// shared small-integer range, negatives and the int64 edges, in far fewer +// allocations. +func TestAppendIntegers(t *testing.T) { + for _, lo := range []int64{-300, -17, 0, 250, 1000, math.MaxInt64 - 299, math.MinInt64} { + const n = 300 + got := AppendIntegers(nil, lo, n) + if len(got) != n { + t.Fatalf("lo=%d: %d items", lo, len(got)) + } + for i, it := range got { + want := NewInteger(lo + int64(i)) + a := it.(*Atomic) + if a.Type != TypeInteger || a.String() != want.String() || a.Rat().Cmp(want.Rat()) != 0 || a.Int64() != lo+int64(i) { + t.Fatalf("lo=%d item %d: got %v want %v", lo, i, a, want) + } + } + } + if got := AppendIntegers(nil, math.MaxInt64, 1); got[0].(*Atomic).Int64() != math.MaxInt64 { + t.Error("MaxInt64") + } + chunked := testing.AllocsPerRun(5, func() { AppendIntegers(nil, 1000, 1280) }) + if chunked > 100 { // NewInteger per value is 3 x 1280 + t.Errorf("%v allocations for 1280 integers", chunked) + } +} diff --git a/xpath/operators.go b/xpath/operators.go index c42133d4..d00c502e 100644 --- a/xpath/operators.go +++ b/xpath/operators.go @@ -873,12 +873,7 @@ func (e *BinaryOp) evalRange(ctx *Context) (xdm.Sequence, error) { // would narrow each value to anyway. Testing v against the end before // incrementing it keeps a range ending at MaxInt64 from overflowing. if lo.IsInt64() && hi.IsInt64() { - for v, end := lo.Int64(), hi.Int64(); ; v++ { - out = append(out, xdm.NewInteger(v)) - if v == end { - return out, nil - } - } + return xdm.AppendIntegers(out, lo.Int64(), count), nil } // Stepping a big.Int keeps the arbitrary precision through the whole // range: the values are what the expression named, not a wrapped From 4b0b564cd47a47cce2499951ceda2218b4605f55 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:26:40 -0400 Subject: [PATCH 77/98] xslt: walk namespace frames instead of building the in-scope map per element --- xslt/instructions.go | 18 ++++-------------- xslt/nsscope_alloc_test.go | 35 +++++++++++++++++++++++++++++++++++ xslt/serialize.go | 21 +++++++++------------ 3 files changed, 48 insertions(+), 26 deletions(-) create mode 100644 xslt/nsscope_alloc_test.go diff --git a/xslt/instructions.go b/xslt/instructions.go index 5f938910..a641e3f8 100644 --- a/xslt/instructions.go +++ b/xslt/instructions.go @@ -531,21 +531,11 @@ func copyDocumentNode(n *xdm.Node) *xdm.Node { // Declarations the element already carries are left alone: they are the // nearest ones, and an inherited binding for the same prefix is masked. func inheritNamespaces(dst, src *xdm.Node) { - have := map[string]bool{} - for prefix := range dst.DeclaredNamespaces() { - have[prefix] = true - } - scope := src.InScopeNamespaces() - prefixes := make([]string, 0, len(scope)) - for p := range scope { - prefixes = append(prefixes, p) - } - sort.Strings(prefixes) - for _, p := range prefixes { - if p == "xml" || have[p] { - continue + var arr [32]nsBinding + for _, b := range scopeBindings(src, arr[:0]) { + if !declares(dst, b.prefix) { + dst.AddNamespace(b.prefix, b.uri) } - dst.AddNamespace(p, scope[p]) } } diff --git a/xslt/nsscope_alloc_test.go b/xslt/nsscope_alloc_test.go new file mode 100644 index 00000000..b48bf9c4 --- /dev/null +++ b/xslt/nsscope_alloc_test.go @@ -0,0 +1,35 @@ +package xslt + +import ( + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// inheritNamespaces and rootNamespaces ran once per copied or serialized +// element and each built InScopeNamespaces' map plus a set of the element's +// own prefixes (about 10 allocations). They now walk the ancestors' frames. +func TestNamespaceScopeWalkDoesNotBuildMaps(t *testing.T) { + doc, err := xdm.ParseString(``+ + ``, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + c := doc.Root.FirstChild().FirstChild().FirstChild() + if c.Name().Local != "c" { + t.Fatalf("picked %s", c.Name().Local) + } + rn := rootNamespaces(c) + if len(rn) != 4 || rn[0].prefix != "" || rn[1].prefix != "p" || + rn[2].prefix != "q" || rn[3].prefix != "r" { + t.Errorf("rootNamespaces = %v, want prefixes \"\", p, q, r", rn) + } + if n := testing.AllocsPerRun(20, func() { _ = rootNamespaces(c) }); n > 2 { + t.Errorf("rootNamespaces made %.0f allocations, want <= 2", n) + } + cp := xdm.Copy(c) + inheritNamespaces(cp, c) + if cp.NumNamespaceDecls() != 4 { + t.Errorf("copy has %d declarations, want 4", cp.NumNamespaceDecls()) + } +} diff --git a/xslt/serialize.go b/xslt/serialize.go index f1610440..50cb3ab7 100644 --- a/xslt/serialize.go +++ b/xslt/serialize.go @@ -5,7 +5,7 @@ import ( "bytes" "fmt" "io" - "sort" + "slices" "strconv" "strings" "sync" @@ -1198,15 +1198,12 @@ func (s *serializer) ownNamespaceDecl(n *xdm.Node, base int, prefix, uri string) // otherwise the union is sorted by prefix, which is the order a literal // result element copies its bindings in. func rootNamespaces(n *xdm.Node) []nsPair { - scope := n.InScopeNamespaces() - own := make(map[string]bool, n.NumNamespaceDecls()) - for prefix := range n.DeclaredNamespaces() { - own[prefix] = true - } - var out []nsPair - for p, uri := range scope { - if p != "xml" && !own[p] { - out = append(out, nsPair{p, uri}) + var arr [32]nsBinding + scope := scopeBindings(n, arr[:0]) + out := make([]nsPair, 0, len(scope)+n.NumNamespaceDecls()) + for _, b := range scope { + if !declares(n, b.prefix) { + out = append(out, nsPair{b.prefix, b.uri}) } } inherited := len(out) @@ -1214,8 +1211,8 @@ func rootNamespaces(n *xdm.Node) []nsPair { out = append(out, nsPair{prefix, uri}) } if inherited > 0 { - sort.SliceStable(out, func(i, j int) bool { - return out[i].prefix < out[j].prefix + slices.SortStableFunc(out, func(a, b nsPair) int { + return strings.Compare(a.prefix, b.prefix) }) } return out From 469c95e03188c91303f797b9c28d9cae4386fa0d Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:27:46 -0400 Subject: [PATCH 78/98] xslt: test array emptiness lazily, stopping at the first deciding leaf --- xslt/onempty.go | 49 ++++++++++++++++++++++++++++----------- xslt/onempty_lazy_test.go | 34 +++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 14 deletions(-) create mode 100644 xslt/onempty_lazy_test.go diff --git a/xslt/onempty.go b/xslt/onempty.go index 728f61b7..16502eeb 100644 --- a/xslt/onempty.go +++ b/xslt/onempty.go @@ -119,13 +119,7 @@ func deemedEmpty(it xdm.Item) bool { case *xdm.ArrayItem: // "an array where the result of flattening the array is either an // empty sequence, or a sequence in which every item is deemed empty". - // Flatten is iterative, so a deeply nested array cannot overflow the stack. - for _, sub := range xdm.Flatten(xdm.Sequence{v}) { - if !deemedEmpty(sub) { - return false - } - } - return true + return everyLeaf(v, deemedEmpty) case *xdm.Atomic: return v.String() == "" } @@ -134,6 +128,39 @@ func deemedEmpty(it xdm.Item) bool { return false } +// everyLeaf reports whether ok holds for every item of a, flattened, and stops +// at the first that fails. It walks with an explicit stack, as xdm.Flatten +// does, so a deeply nested array cannot overflow the Go stack, but without +// materialising the whole flattened sequence first: an array whose first leaf +// is content is answered after that leaf. +func everyLeaf(a *xdm.ArrayItem, ok func(xdm.Item) bool) bool { + type frame struct { + arr *xdm.ArrayItem + m, i int // member (1-based), and item within it + } + stack := []frame{{arr: a, m: 1}} + for len(stack) > 0 { + f := &stack[len(stack)-1] + if f.m > f.arr.Len() { + stack = stack[:len(stack)-1] + continue + } + mem, err := f.arr.Member(f.m) + if err != nil || f.i == len(mem) { + f.m, f.i = f.m+1, 0 + continue + } + it := mem[f.i] + f.i++ + if arr, isArr := it.(*xdm.ArrayItem); isArr { + stack = append(stack, frame{arr: arr, m: 1}) + } else if !ok(it) { + return false + } + } + return true +} + // vacuous is 8.4.2's definition, which decides whether xsl:on-empty fires. // // The list is closed and shorter than deemed-empty's: a zero-length text node, @@ -159,13 +186,7 @@ func vacuous(it xdm.Item) bool { return false } case *xdm.ArrayItem: - // Flatten is iterative, so a deeply nested array cannot overflow the stack. - for _, sub := range xdm.Flatten(xdm.Sequence{v}) { - if !vacuous(sub) { - return false - } - } - return true + return everyLeaf(v, vacuous) case *xdm.Atomic: return v.String() == "" } diff --git a/xslt/onempty_lazy_test.go b/xslt/onempty_lazy_test.go new file mode 100644 index 00000000..7d4b6ad5 --- /dev/null +++ b/xslt/onempty_lazy_test.go @@ -0,0 +1,34 @@ +package xslt + +import ( + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// vacuous and deemedEmpty used to flatten the whole array before testing it, +// 16 B a leaf, so a 100,000-member array whose first member was content still +// copied all of them. They now stop at the first leaf that decides. +func TestEmptinessOfArrayStopsAtFirstDecidingLeaf(t *testing.T) { + const n = 100000 + members := make([]xdm.Sequence, n) + members[0] = xdm.Sequence{xdm.NewString("x")} + for i := 1; i < n; i++ { + members[i] = xdm.Sequence{xdm.NewString("")} + } + arr := xdm.NewArray(members...) + if vacuous(arr) || deemedEmpty(arr) { + t.Fatal("an array holding \"x\" is content") + } + for name, f := range map[string]func(xdm.Item) bool{"vacuous": vacuous, "deemedEmpty": deemedEmpty} { + if a := testing.AllocsPerRun(5, func() { f(arr) }); a > 4 { + t.Errorf("%s made %.0f allocations on an array decided by its first leaf", name, a) + } + } + // Nested, all empty: both are true, and the answer survives nesting. + inner := xdm.NewArray(xdm.Sequence{xdm.NewString("")}, xdm.Sequence{}) + outer := xdm.NewArray(xdm.Sequence{inner}, xdm.Sequence{xdm.NewArray()}) + if !vacuous(outer) || !deemedEmpty(outer) { + t.Error("nested all-empty array should be vacuous and deemed-empty") + } +} From a3d8ef667157ebc15d2aa3ed9adf94178ebf8ebb Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:28:01 -0400 Subject: [PATCH 79/98] xslt: carry xsl:iterate parameters in slices and charge xsl:next-iteration values --- xslt/compile_instr.go | 90 ++++++++++++++++++++++++++++--------- xslt/iterateparams_test.go | 74 ++++++++++++++++++++++++++++++ xslt/resultdocscale_test.go | 16 ++++--- 3 files changed, 151 insertions(+), 29 deletions(-) create mode 100644 xslt/iterateparams_test.go diff --git a/xslt/compile_instr.go b/xslt/compile_instr.go index 0f9d294f..29c32d04 100644 --- a/xslt/compile_instr.go +++ b/xslt/compile_instr.go @@ -2027,8 +2027,12 @@ type breakSignal struct{} func (breakSignal) Error() string { return "xsl:break" } // nextIterationSignal unwinds to the end of one turn of the loop, carrying the -// parameter values for the next. -type nextIterationSignal struct{ params map[string]xdm.Sequence } +// parameter values for the next. vals runs parallel to the with-params of +// from, whose slots say which xsl:param of the loop each one is. +type nextIterationSignal struct { + vals []xdm.Sequence + from *nextIterationInstr +} func (nextIterationSignal) Error() string { return "xsl:next-iteration" } @@ -2057,19 +2061,31 @@ func (i *breakInstr) Execute(rt *runtime, out *outputBuilder) error { return breakSignal{} } -type nextIterationInstr struct{ params []*Variable } +type nextIterationInstr struct { + params []*Variable + // slots[j] is the position among the enclosing xsl:iterate's xsl:param + // children of params[j]; byDecl lists the j in that declaration order. + slots []int + byDecl []int +} func (i *nextIterationInstr) Execute(rt *runtime, out *outputBuilder) error { - vals := map[string]xdm.Sequence{} - for _, p := range i.params { + vals := make([]xdm.Sequence, len(i.params)) + for j, p := range i.params { v, err := evalVariable(p, rt) if err != nil { return fmt.Errorf("evaluating parameter $%s of xsl:next-iteration: %w", p.Name.Lexical(), err) } - vals[p.Name.Clark()] = v + // The value is carried into every later turn, so a binding larger + // than the item bound is refused here rather than held. + if err := rt.ctx.ItemsRefused(int64(len(v))); err != nil { + return fmt.Errorf("parameter $%s of xsl:next-iteration: %w", + p.Name.Lexical(), err) + } + vals[j] = v } - return nextIterationSignal{params: vals} + return nextIterationSignal{vals: vals, from: i} } // appendSequence writes a sequence to an output builder, the same way @@ -2107,21 +2123,21 @@ func (i *iterateInstr) Execute(rt *runtime, out *outputBuilder) error { // The carried state starts at the declared defaults, which are evaluated // in the focus of the xsl:iterate instruction itself rather than of any // item, because on the first turn no item has been reached yet. - carried := map[string]xdm.Sequence{} - for _, p := range i.params { + carried := make([]xdm.Sequence, len(i.params)) + for k, p := range i.params { v, err := evalVariable(p, rt) if err != nil { return fmt.Errorf("evaluating parameter $%s of xsl:iterate: %w", p.Name.Lexical(), err) } - carried[p.Name.Clark()] = v + carried[k] = v } broke := false for idx, it := range seq { sub := rt.withCurrent(it, idx+1, len(seq)) mark := sub.localsMark() - for _, p := range i.params { - sub = sub.bindLocal(p.Name, carried[p.Name.Clark()]) + for k, p := range i.params { + sub = sub.bindLocal(p.Name, carried[k]) } err := execSequence(i.body, sub, out) sub.localsPop(mark) @@ -2147,13 +2163,10 @@ func (i *iterateInstr) Execute(rt *runtime, out *outputBuilder) error { // XTTE0590 reported is the first declared, on every run. // XTSE3130 has already refused a with-param naming no // xsl:param, so nothing supplied is skipped. - if len(nx.params) > 0 { - for _, p := range i.params { - k := p.Name.Clark() - v, ok := nx.params[k] - if !ok { - continue - } + if len(nx.vals) > 0 { + for _, j := range nx.from.byDecl { + k := nx.from.slots[j] + p, v := i.params[k], nx.vals[j] conv, err := p.asType.convertAsLazy(v, func() string { return "parameter $" + p.Name.Lexical() + " of xsl:iterate" }, "XTTE0590") @@ -2179,8 +2192,8 @@ func (i *iterateInstr) Execute(rt *runtime, out *outputBuilder) error { // xsl:number with no context node is XTTE0990. done := rt.withFocus(nil, 0, 0) defer done.localsPop(done.localsMark()) - for _, p := range i.params { - done = done.bindLocal(p.Name, carried[p.Name.Clark()]) + for k, p := range i.params { + done = done.bindLocal(p.Name, carried[k]) } if i.onCompletionSelect != nil { vals, err := i.onCompletionSelect.Eval(done.ctx) @@ -2273,7 +2286,40 @@ func (c *compiler) compileNextIteration(n *xdm.Node, ns *nsResolver) (Instructio if err != nil { return nil, err } - return &nextIterationInstr{params: params}, nil + instr := &nextIterationInstr{params: params} + // The innermost xsl:iterate's parameters are known by now: compileIterate + // reads them before it compiles the body this instruction is in. + var declared []xdm.QName + if it := enclosingIterate(n); it != nil { + for _, c := range it.ChildElements() { + if isXSL(c, "param") { + qn, _ := paramName(c) + declared = append(declared, qn) + } + } + } + for _, p := range params { + slot := -1 + for k, qn := range declared { + if qn.URI == p.Name.URI && qn.Local == p.Name.Local { + slot = k + break + } + } + if slot < 0 { + return nil, fmt.Errorf("XTSE3130: xsl:with-param %s names no "+ + "xsl:param of the containing xsl:iterate", p.Name.Lexical()) + } + instr.slots = append(instr.slots, slot) + } + instr.byDecl = make([]int, len(params)) + for j := range instr.byDecl { + instr.byDecl[j] = j + } + sort.Slice(instr.byDecl, func(a, b int) bool { + return instr.slots[instr.byDecl[a]] < instr.slots[instr.byDecl[b]] + }) + return instr, nil } // textParent is the stylesheet element a text node of a sequence constructor diff --git a/xslt/iterateparams_test.go b/xslt/iterateparams_test.go new file mode 100644 index 00000000..817c8032 --- /dev/null +++ b/xslt/iterateparams_test.go @@ -0,0 +1,74 @@ +package xslt + +import ( + "context" + "errors" + goruntime "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +func iterateSheet(t *testing.T, body string) *Stylesheet { + t.Helper() + st, err := xdm.ParseString(` + + `+body+``, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + c, err := Compile(st.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + return c +} + +// A value handed on by xsl:next-iteration is held for every later turn, so +// one larger than the item bound is refused rather than carried. +func TestIterateNextIterationParamIsCharged(t *testing.T) { + c := iterateSheet(t, ` + + + + `) + src, _ := xdm.ParseString("", xdm.ParseOptions{}) + _, err := c.Transform(context.Background(), src.Root, TransformOptions{MaxItems: 1000}) + if err == nil || !errors.Is(err, xdm.ErrResourceLimit) || + !strings.Contains(err.Error(), "xsl:next-iteration") { + t.Fatalf("err = %v, want a resource limit naming xsl:next-iteration", err) + } +} + +// The parameters are carried in slices indexed by declaration order, whatever +// order xsl:with-param names them in, and a turn costs no map. +func TestIterateParamsCarryInDeclarationOrder(t *testing.T) { + c := iterateSheet(t, ` + + + + + + `) + src, _ := xdm.ParseString("", xdm.ParseOptions{}) + var m0, m1 goruntime.MemStats + goruntime.ReadMemStats(&m0) + res, err := c.Transform(context.Background(), src.Root, TransformOptions{}) + goruntime.ReadMemStats(&m1) + if err != nil { + t.Fatal(err) + } + var b strings.Builder + if err := res.Serialize(&b); err != nil { + t.Fatal(err) + } + if b.String() != "100000 200000" { + t.Fatalf("got %q", b.String()) + } + // 800 bytes a turn with slices; 1119 when each turn built a map. + if per := (m1.TotalAlloc - m0.TotalAlloc) / 100000; per > 950 { + t.Fatalf("a turn allocated %d bytes", per) + } +} diff --git a/xslt/resultdocscale_test.go b/xslt/resultdocscale_test.go index 95f91813..c0ef9877 100644 --- a/xslt/resultdocscale_test.go +++ b/xslt/resultdocscale_test.go @@ -44,16 +44,18 @@ func TestResultDocumentCollisionCheckIsLinear(t *testing.T) { return time.Since(start) } best := func(n int) time.Duration { - a, b := run(n), run(n) - if b < a { - return b + best := run(n) + for i := 0; i < 4; i++ { + if d := run(n); d < best { + best = d + } } - return a + return best } - small, large := best(3000), best(12000) + small, large := best(6000), best(24000) // 4x the documents: about 4x the time when linear, about 16x when not. - if large > 9*small { - t.Fatalf("12000 documents took %v against %v for 3000", large, small) + if large > 12*small { + t.Fatalf("24000 documents took %v against %v for 6000", large, small) } // A repeated href is still XTDE1490. From 44229aa084b290c548d55aeaf8d055c9a0d8cdf8 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:29:44 -0400 Subject: [PATCH 80/98] relaxng: merge choice chains alternative by alternative so (a?){n} a{n} validates --- relaxng/choice_dedupe_test.go | 30 +++++++++++++++++++++++++++ relaxng/derive.go | 38 +++++++++++++++++++++++++++++++++-- relaxng/pattern_size_test.go | 2 +- relaxng/validate.go | 4 ++-- 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/relaxng/choice_dedupe_test.go b/relaxng/choice_dedupe_test.go index 780590b2..23d39cfa 100644 --- a/relaxng/choice_dedupe_test.go +++ b/relaxng/choice_dedupe_test.go @@ -59,3 +59,33 @@ func TestChoiceMergesEqualAlternatives(t *testing.T) { t.Error("choice(value, value) merged two values") } } + +// (a?){n} a{n} against n is valid. The derivative is a choice of +// choices whose chains share nearly every alternative; merging only a +// whole chain left the copies in, and from n = 13 the pattern passed the +// 100,000-node bound and the valid document was refused. xmllint agrees +// that it validates for n = 12, 13, 18 and 30. +func TestOptionalsBeforeRequiredStayValid(t *testing.T) { + for _, n := range []int{12, 13, 18, 30} { + var b strings.Builder + b.WriteString(``) + for i := 0; i < n; i++ { + b.WriteString(``) + } + for i := 0; i < n; i++ { + b.WriteString(``) + } + b.WriteString(``) + s, err := compileString(t, b.String()) + if err != nil { + t.Fatalf("compile: %v", err) + } + doc, err := xdm.ParseString(""+strings.Repeat("", n)+"", xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + if err := s.Validate(doc.Root); err != nil { + t.Errorf("n=%d: %v", n, err) + } + } +} diff --git a/relaxng/derive.go b/relaxng/derive.go index bfd26766..e848d38e 100644 --- a/relaxng/derive.go +++ b/relaxng/derive.go @@ -210,22 +210,56 @@ func (pb *patBuilder) choice(a, b pattern) pattern { if patEq(a, b) || pb.inChoice(a, b) { return a } + // A choice on the right is merged alternative by alternative. Taking the + // derivative of a choice of choices yields chains that share most of their + // alternatives, and testing the whole chain against a only finds a chain + // that is identical: (a?){13} a{13} then grew past the pattern bound while + // holding a few dozen distinct alternatives. + // + // The merge compares every pair of alternatives, so it is skipped when + // the chains are long enough to make that dearer than the copies it + // saves: alternatives that never compare equal (values, data) would + // otherwise buy a quadratic cost per choice from a hostile document. + // ponytail: a fixed pair budget; a hash of the alternatives if a schema + // needs more. + if c, ok := b.(*choicePat); ok && altCount(a)*altCount(b) <= maxMergePairs { + return pb.choice(pb.choice(a, c.Left), c.Right) + } if pb != nil { return pb.mk(kChoice, a, b) } return newChoicePat(a, b) } +// maxMergePairs bounds the pairs of alternatives choice compares when it +// merges one chain into another. +const maxMergePairs = 1 << 10 + +// altCount is the number of alternatives in a chain of choices. +func altCount(p pattern) int { + n := 1 + for { + c, ok := unstatic(p).(*choicePat) + if !ok { + return n + } + n += altCount(c.Right) + p = c.Left + } +} + // inChoice reports whether b is already one of a's alternatives. A chain of // choices is built leftwards, so the alternatives are the Right of each link // and the Left of the last. func (pb *patBuilder) inChoice(a, b pattern) bool { + // Equal patterns have equal sizes, so a size mismatch skips the walk. + sb := patternSize(b) for { c, ok := unstatic(a).(*choicePat) if !ok { - return patEq(a, b) + return patternSize(a) == sb && patEq(a, b) } - if patEq(c.Right, b) { + if patternSize(c.Right) == sb && patEq(c.Right, b) { return true } a = c.Left diff --git a/relaxng/pattern_size_test.go b/relaxng/pattern_size_test.go index 5bc3c760..8775fefc 100644 --- a/relaxng/pattern_size_test.go +++ b/relaxng/pattern_size_test.go @@ -110,7 +110,7 @@ func TestPlainOneOrMoreStillValidates(t *testing.T) { // at twelve, which still finishes, rather than hanging at fourteen. func TestAttributePatternSizeIsBounded(t *testing.T) { const sch = ` -` +` schTree, err := xdm.ParseString(sch, xdm.ParseOptions{}) if err != nil { t.Fatal(err) diff --git a/relaxng/validate.go b/relaxng/validate.go index d93ffcd5..0ed1518e 100644 --- a/relaxng/validate.go +++ b/relaxng/validate.go @@ -111,8 +111,8 @@ func (s *Schema) ValidateWithOptions(doc *xdm.Node, opts ValidateOptions) error return &Error{ Path: v.deepPath, Message: fmt.Sprintf( - "the derivative pattern exceeds %d nodes; a oneOrMore nested "+ - "inside a oneOrMore grows the pattern multiplicatively in "+ + "the derivative pattern exceeds %d nodes; nested repetition "+ + "or optional content grows the pattern multiplicatively in "+ "the number of children", maxPattern), } } From a6f2eaa2fcb687df0a4dfade32df13c09d02da28 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:29:47 -0400 Subject: [PATCH 81/98] xslt: parse fn:doc files as they are read, not through ReadAll and string() --- xslt/resolver.go | 115 ++++++++++++++++++++++++++--------- xslt/resolver_stream_test.go | 51 ++++++++++++++++ 2 files changed, 138 insertions(+), 28 deletions(-) create mode 100644 xslt/resolver_stream_test.go diff --git a/xslt/resolver.go b/xslt/resolver.go index d76c95e8..ccfc5b82 100644 --- a/xslt/resolver.go +++ b/xslt/resolver.go @@ -227,6 +227,17 @@ func NewFileResolver(roots ...string) (*FileResolver, error) { // permitted directories, and it is what decides *which* root a path belongs to. // This is the enforcement, not the diagnosis. func (r *FileResolver) readConfined(path string) ([]byte, error) { + var data []byte + err := r.openConfined(path, func(f *os.File) (err error) { + data, err = r.readLimited(f, path) + return err + }) + return data, err +} + +// openConfined is readConfined up to the read: it opens path the same way and +// hands the open file to use, closing it afterwards. +func (r *FileResolver) openConfined(path string, use func(f *os.File) error) error { for _, root := range r.Roots { absRoot, err := filepath.Abs(root) if err != nil { @@ -260,12 +271,12 @@ func (r *FileResolver) readConfined(path string) ([]byte, error) { f, err := rt.Open(filepath.ToSlash(rel)) if err != nil { rt.Close() - return nil, err + return err } - data, err := r.readLimited(f, path) + err = use(f) f.Close() rt.Close() - return data, err + return err } // No root contains it. A resolver with no roots at all is the documented // unconfined case, and reaching here means resolvePath admitted a path @@ -274,12 +285,12 @@ func (r *FileResolver) readConfined(path string) ([]byte, error) { if len(r.Roots) == 0 { f, err := os.Open(path) if err != nil { - return nil, err + return err } defer f.Close() - return r.readLimited(f, path) + return use(f) } - return nil, fmt.Errorf("%q is outside the permitted directories %v", + return fmt.Errorf("%q is outside the permitted directories %v", path, r.Roots) } @@ -292,24 +303,10 @@ func (r *FileResolver) readConfined(path string) ([]byte, error) { // outcome this library declines to produce. The refusal names the limit so that // a caller who wants the file raises MaxBytes rather than guessing. func (r *FileResolver) readLimited(f io.Reader, path string) ([]byte, error) { - max := r.MaxBytes - if max == 0 { - max = DefaultMaxResourceBytes - } + max, lim := r.byteLimit() if max < 0 { return io.ReadAll(f) } - // One byte over the limit is read deliberately, so that a file of exactly - // the maximum size is accepted and one byte more is distinguishable from - // it. The increment saturates: max+1 overflows to a negative limit at - // math.MaxInt64, and io.LimitReader reads a negative limit as "nothing - // left", so the largest limit a caller can name would silently return an - // empty file. An empty read is worse than a refusal, because nothing - // downstream can tell it from a genuinely empty resource. - lim := max - if lim < math.MaxInt64 { - lim++ - } data, err := io.ReadAll(io.LimitReader(f, lim)) if err != nil { return nil, err @@ -320,6 +317,40 @@ func (r *FileResolver) readLimited(f io.Reader, path string) ([]byte, error) { return data, nil } +// byteLimit returns the resolver's byte limit (negative: none) and the number +// of bytes to read to enforce it. +// +// One byte over the limit is read deliberately, so that a file of exactly +// the maximum size is accepted and one byte more is distinguishable from +// it. The increment saturates: max+1 overflows to a negative limit at +// math.MaxInt64, and io.LimitReader reads a negative limit as "nothing +// left", so the largest limit a caller can name would silently return an +// empty file. An empty read is worse than a refusal, because nothing +// downstream can tell it from a genuinely empty resource. +func (r *FileResolver) byteLimit() (max, lim int64) { + max = r.MaxBytes + if max == 0 { + max = DefaultMaxResourceBytes + } + lim = max + if lim >= 0 && lim < math.MaxInt64 { + lim++ + } + return max, lim +} + +// countedReader counts the bytes read through it. +type countedReader struct { + r io.Reader + n int64 +} + +func (c *countedReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + // resolvePath turns an href into a filesystem path inside an allowed root. func (r *FileResolver) resolvePath(href, base string) (string, error) { // An empty URI reference denotes the document the reference appears in, @@ -649,10 +680,6 @@ func (r *FileResolver) publish(path string, tree *xdm.Tree, size int64) { // that used to run under r.mu. func (r *FileResolver) parseUncached( path string, trackPos bool, b *xdm.EntityBudget) (*xdm.Tree, int64, error) { - data, err := r.readConfined(path) - if err != nil { - return nil, 0, err - } // A file: URI rather than the bare path. This base URI is stamped on // every element of the module and is what fn:document, fn:resolve-uri // and fn:static-base-uri resolve against, and those are defined over @@ -678,11 +705,43 @@ func (r *FileResolver) parseUncached( // The expansion is charged against the caller's shared allowance rather // than a fresh one per file. A nil budget leaves the per-document ceiling // in place. See ResolveModuleWith. - tree, err := xdm.ParseString(string(data), opts.WithEntityBudget(b)) + // + // The file is parsed as it is read, so a large document is never held + // as bytes and again as a string beside its tree (62 MB: 477 MB peak + // through fn:doc against 245 MB as the source). The byte limit is + // enforced as readLimited does: a file whose size is already known to + // exceed it is refused before anything is parsed, and one that grows + // past it while being read is refused whatever the parser made of the + // truncated text, with the same message. + var ( + tree *xdm.Tree + size int64 + ) + err := r.openConfined(path, func(f *os.File) error { + max, lim := r.byteLimit() + tooBig := fmt.Errorf("%s exceeds the %d byte limit", path, max) + var src io.Reader = f + if max >= 0 { + if fi, err := f.Stat(); err == nil && fi.Size() > max { + return tooBig + } + src = io.LimitReader(f, lim) + } + cr := &countedReader{r: src} + t, err := xdm.Parse(cr, opts.WithEntityBudget(b)) + if max >= 0 && cr.n > max { + return tooBig + } + if err != nil { + return fmt.Errorf("parsing %s: %w", path, err) + } + tree, size = t, cr.n + return nil + }) if err != nil { - return nil, 0, fmt.Errorf("parsing %s: %w", path, err) + return nil, 0, err } - return tree, int64(len(data)), nil + return tree, size, nil } // fileURIToPath turns a file: URI back into a filesystem path, and leaves diff --git a/xslt/resolver_stream_test.go b/xslt/resolver_stream_test.go new file mode 100644 index 00000000..51b3b07e --- /dev/null +++ b/xslt/resolver_stream_test.go @@ -0,0 +1,51 @@ +package xslt + +import ( + "fmt" + "os" + "path/filepath" + goruntime "runtime" + "strings" + "testing" +) + +// A document read through fn:doc is parsed as it is read. It used to be read +// whole by io.ReadAll (about two copies while it grew) and then converted with +// string(data), so a 62 MB file allocated 121 MB + 59 MB beyond its tree. +// Allocated bytes are the measure, not wall time: the document is mostly a +// single text run, whose tree costs about its own size, so the whole call must +// stay below 3x the file. +func TestResolveDocumentDoesNotBufferTheFile(t *testing.T) { + const size = 16 << 20 + dir := t.TempDir() + var sb strings.Builder + sb.WriteString("") + for sb.Len() < size { + fmt.Fprintf(&sb, "%s", strings.Repeat("x", 100)) + } + sb.WriteString("") + path := filepath.Join(dir, "big.xml") + if err := os.WriteFile(path, []byte(sb.String()), 0o600); err != nil { + t.Fatal(err) + } + fileSize := int64(sb.Len()) + sb.Reset() + r, err := NewFileResolver(dir) + if err != nil { + t.Fatal(err) + } + var m0, m1 goruntime.MemStats + goruntime.GC() + goruntime.ReadMemStats(&m0) + tree, err := r.ResolveDocument(path, "") + if err != nil { + t.Fatal(err) + } + goruntime.ReadMemStats(&m1) + goruntime.KeepAlive(tree) + t.Logf("file %d, allocated %d", fileSize, m1.TotalAlloc-m0.TotalAlloc) + if got := int64(m1.TotalAlloc - m0.TotalAlloc); got > 3*fileSize { + t.Errorf("ResolveDocument allocated %d bytes for a %d byte file, want <= %d", + got, fileSize, 3*fileSize) + } +} From f49ff9202f47364c071975b78844ca716677bd29 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:31:56 -0400 Subject: [PATCH 82/98] xslt: FileResolver does not evict to admit an entry that does not fit the byte bound --- xslt/resolver.go | 13 +++++++++++++ xslt/resolver_cache_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/xslt/resolver.go b/xslt/resolver.go index ccfc5b82..af1b301c 100644 --- a/xslt/resolver.go +++ b/xslt/resolver.go @@ -661,6 +661,19 @@ func (r *FileResolver) publish(path string, tree *xdm.Tree, size int64) { r.cacheBytes -= r.sizes[path] delete(r.sizes, path) delete(r.cache, path) + // An entry that fits the byte bound alone but not beside what the cache + // holds is not admitted, and nothing is evicted for it. Evicting at random + // to make room kept throwing out documents that were about to be read + // again: four 9 MB files cycled against the 32 MB bound hit 22 times in + // 48, where keeping the three that fit hits 3 in 4. The cost is that the + // rejected document is parsed again on its next request, which gives a + // different tree, and that a resolver whose byte bound filled early keeps + // its first documents; the bound on memory is unchanged. A document that + // alone exceeds the bound is still admitted, after everything else has + // been evicted, because it can never share the cache. + if size <= resolverCacheMaxBytes && r.cacheBytes+size > resolverCacheMaxBytes { + return + } for len(r.cache) > 0 && (len(r.cache) >= resolverCacheMax || r.cacheBytes+size > resolverCacheMaxBytes) { diff --git a/xslt/resolver_cache_test.go b/xslt/resolver_cache_test.go index eab6a8e4..e0b8c75e 100644 --- a/xslt/resolver_cache_test.go +++ b/xslt/resolver_cache_test.go @@ -218,3 +218,32 @@ func TestResolverCacheIsBoundedByBytes(t *testing.T) { t.Errorf("only %d documents retained; the bound evicted far more than it needed", len(res.cache)) } } + +// An entry that does not fit beside the cache's contents is not admitted, and +// nothing is evicted for it: four 9 MB files against the 32 MB bound keep the +// three that fit, rather than evicting one at random for each arrival. +func TestResolverCacheDoesNotEvictToAdmitAnEntryThatDoesNotFit(t *testing.T) { + tree, err := xdm.ParseString("", xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + r := &FileResolver{} + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range []string{"a", "b", "c", "d"} { + r.publish(p, tree, 9<<20) + } + for _, p := range []string{"a", "b", "c"} { + if _, ok := r.cache[p]; !ok { + t.Errorf("%s was evicted to make room for d", p) + } + } + if _, ok := r.cache["d"]; ok { + t.Error("d was admitted although 36 MB does not fit in 32 MB") + } + // A document over the bound alone is still admitted, alone. + r.publish("big", tree, resolverCacheMaxBytes+1) + if _, ok := r.cache["big"]; !ok || len(r.cache) != 1 { + t.Errorf("oversize entry: cache holds %d entries, want only big", len(r.cache)) + } +} From be5afc3e0aeeb7cd8ccf9f33d35fa44afb7e11f7 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:33:16 -0400 Subject: [PATCH 83/98] xdm: store each DTD default value once and alias it in every element --- xdm/dtd_defaults_test.go | 61 +++++++++++++++++++++++++++++++++++++++- xdm/parse.go | 5 ++++ xdm/record.go | 45 ++++++++++++++++++++++++++--- xdm/wellformed.go | 8 +++++- 4 files changed, 113 insertions(+), 6 deletions(-) diff --git a/xdm/dtd_defaults_test.go b/xdm/dtd_defaults_test.go index bc58c06e..0882b353 100644 --- a/xdm/dtd_defaults_test.go +++ b/xdm/dtd_defaults_test.go @@ -1,6 +1,11 @@ package xdm -import "testing" +import ( + "fmt" + "runtime" + "strings" + "testing" +) // An ATTLIST default is supplied to every matching element, including a // namespace declaration — "xmlns:p CDATA #FIXED '...'" is how a DTD supplies @@ -218,3 +223,57 @@ func TestAttListFirstDeclarationBinds(t *testing.T) { t.Errorf("s/@a = %q, want z1: the first-binds rule is per element", got) } } + +// A default is declared once and applies to every element. Its value used to +// be copied into the tree for each element that received it, so 2 MB of input +// retained 1.9 GB (250 attributes of 8,000 bytes on 1,000 elements). The +// value is stored once and aliased: here 20 defaults of 4,000 bytes on 200 +// elements need 80 KB, and 16 MB when copied. +func TestAttListDefaultValuesAreShared(t *testing.T) { + const attrs, elems, size = 20, 200, 4000 + var dtd, doc strings.Builder + dtd.WriteString("") + for i := 0; i < attrs; i++ { + // Half are NMTOKENS, whose value is collapsed per element. + typ, val := "CDATA", strings.Repeat("x", size) + if i%2 == 1 { + typ, val = "NMTOKENS", strings.Repeat("x ", size/2-1)+"xx" + } + fmt.Fprintf(&dtd, ``, i, typ, val) + } + doc.WriteString("") + for i := 0; i < elems; i++ { + doc.WriteString("") + } + doc.WriteString("") + src := doc.String() + var tree *Tree + var err error + var m0, m1 runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&m0) + tree, err = ParseString(src, ParseOptions{AllowDOCTYPE: true}) + runtime.GC() + runtime.ReadMemStats(&m1) + if err != nil { + t.Fatal(err) + } + if got := int64(m1.HeapAlloc) - int64(m0.HeapAlloc); got > 4<<20 { + t.Errorf("tree retains %d KB, want under 4096 KB", got>>10) + } + n := 0 + for r := range tree.Root.Children() { + for e := range r.Children() { + for a := range e.Attrs() { + if len(a.Value()) != size { + t.Fatalf("attribute %s has %d bytes, want %d", a.Name().Local, len(a.Value()), size) + } + n++ + } + } + } + if n != attrs*elems { + t.Errorf("%d defaulted attributes, want %d", n, attrs*elems) + } + runtime.KeepAlive(tree) +} diff --git a/xdm/parse.go b/xdm/parse.go index 950f861b..25427745 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -639,6 +639,11 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { if attDefaults, err = normalizeAttDefaults(attDefaults, ents, dec.IsVersion11(), wfc); err != nil { return nil, fmt.Errorf("parse XML: %w", err) } + // Each default lands in every matching element; store its + // value once so that they all alias it. + for _, d := range attDefaults { + tree.text.share(d.value) + } if !wfc { declared := ents dec.Undeclared = func(name string) bool { return !declared.declares(name) } diff --git a/xdm/record.go b/xdm/record.go index 17936515..b8844c83 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1544,10 +1544,24 @@ func (n *Node) setOffset(v int32) { // a value handed out as a string never changes under its reader. type textStore struct { blocks [][]byte - cur int // the ordinary block values are appended to, + 1 (0: none) - size int // capacity of the last ordinary block made + cur int32 // the ordinary block values are appended to, + 1 (0: none) + size int32 // capacity of the last ordinary block made + // shared maps a string registered by share to where it is stored, so that + // add hands the same range to every value that is that string. + shared map[sharedKey][2]uint32 } +// sharedKey identifies a string by where it lives: the same storage is the +// same text, because strings are immutable. +type sharedKey struct { + p *byte + n int +} + +// shareMin is the shortest value share and add consult the table for; a short +// one costs less to copy than to look up. +const shareMin = 64 + // Ordinary blocks stop at 32 KB for the reason record chunks do. const ( textShift = 15 @@ -1570,6 +1584,11 @@ func (s *textStore) add(v string) (uint32, uint32) { if v == "" { return 0, 0 } + if s.shared != nil && len(v) >= shareMin { + if r, ok := s.shared[sharedKey{unsafe.StringData(v), len(v)}]; ok { + return r[0], r[1] + } + } // A string is immutable, so a long one is kept as its own block rather // than copied: its capacity is its length, so nothing appends into it. // Every unsafe.String in the module is over append-only bytes (the @@ -1580,6 +1599,24 @@ func (s *textStore) add(v string) (uint32, uint32) { return s.addBytes(unsafe.Slice(unsafe.StringData(v), len(v)), 0) } +// share stores v once and makes every later add of that very string (the same +// storage, not merely equal text) return the stored range. Bytes are only +// appended and extend writes only past the end of its own value, so a range +// held by many values stays valid. +func (s *textStore) share(v string) { + if len(v) < shareMin { + return + } + if s.shared == nil { + s.shared = map[sharedKey][2]uint32{} + } + k := sharedKey{unsafe.StringData(v), len(v)} + if _, ok := s.shared[k]; !ok { + off, n := s.add(v) + s.shared[k] = [2]uint32{off, n} + } +} + // keep stores b, which nothing else writes to, as a block of its own. func (s *textStore) keep(b []byte) (uint32, uint32) { s.newBlock(b) @@ -1597,9 +1634,9 @@ func (s *textStore) addBytes(b []byte, extra int) (uint32, uint32) { return uint32(len(s.blocks)-1) << textShift, uint32(len(b)) } if s.cur == 0 || cap(s.blocks[s.cur-1])-len(s.blocks[s.cur-1]) < need { - s.size = min(max(2*s.size, 64, need), textBlock) + s.size = int32(min(max(2*int(s.size), 64, need), textBlock)) s.newBlock(make([]byte, 0, s.size)) - s.cur = len(s.blocks) + s.cur = int32(len(s.blocks)) } k := s.cur - 1 off := len(s.blocks[k]) diff --git a/xdm/wellformed.go b/xdm/wellformed.go index 700d8362..8230ae11 100644 --- a/xdm/wellformed.go +++ b/xdm/wellformed.go @@ -374,11 +374,17 @@ func normalizeAttTokens(t xml.StartElement, types []attDeclaredType) xml.StartEl if !lexicalIs(a.Name.Space, a.Name.Local, d.attr) && a.Name.Local != d.attr { continue } + // A value already collapsed is left as it is, so that a default + // keeps the storage the text store shares. + c := strings.Join(strings.FieldsFunc(a.Value, func(r rune) bool { return r == ' ' }), " ") + if c == a.Value { + continue + } if attrs == nil { // Copy: the tokeniser's attribute slice is not ours to write. attrs = append([]xml.Attr(nil), t.Attr...) } - attrs[i].Value = strings.Join(strings.FieldsFunc(a.Value, func(r rune) bool { return r == ' ' }), " ") + attrs[i].Value = c } } if attrs != nil { From f3449f742c6823bd5efc0fc0b11355b4e60bf961 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:34:07 -0400 Subject: [PATCH 84/98] cmd/go-xml: drop to GOGC=100 once the live heap passes 128 MB --- cmd/go-xml/gc_test.go | 20 ++++++++++++++++++++ cmd/go-xml/main.go | 44 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/cmd/go-xml/gc_test.go b/cmd/go-xml/gc_test.go index 99744c68..4d62594b 100644 --- a/cmd/go-xml/gc_test.go +++ b/cmd/go-xml/gc_test.go @@ -23,3 +23,23 @@ func TestLowerGCRespectsGOGC(t *testing.T) { t.Errorf("with GOGC=100: GC percent %d, want the environment's 100", got) } } + +// Past 128 MB of live heap the command drops from GOGC=200 to 100; below it +// the setting is left alone. +func TestRelaxGCOnBigLiveHeap(t *testing.T) { + orig := debug.SetGCPercent(cliGCPercent) + defer debug.SetGCPercent(orig) + + if relaxGC(cliGCBigHeap) { + t.Error("relaxed at exactly the threshold, want only past it") + } + if got := debug.SetGCPercent(cliGCPercent); got != cliGCPercent { + t.Errorf("a small heap changed GC percent to %d", got) + } + if !relaxGC(cliGCBigHeap + 1) { + t.Error("did not relax past the threshold") + } + if got := debug.SetGCPercent(cliGCPercent); got != 100 { + t.Errorf("GC percent after a big heap = %d, want 100", got) + } +} diff --git a/cmd/go-xml/main.go b/cmd/go-xml/main.go index 379052d6..b9022508 100644 --- a/cmd/go-xml/main.go +++ b/cmd/go-xml/main.go @@ -11,8 +11,11 @@ import ( "os" "path" "path/filepath" + "runtime" "runtime/debug" + "runtime/metrics" "strings" + "sync" "time" "github.com/knroy/go-xml/v2/xdm" @@ -51,12 +54,51 @@ func main() { // never in the library, whose callers own their process's GC policy. const cliGCPercent = 200 +// cliGCBigHeap is the live heap, in bytes, past which the command drops to +// GOGC=100. At 200 the heap may reach three times what is live, and past this +// size that is what costs: peak RSS on DocBook table-cals.049 was 1,160 MB at +// 200 and 789 MB at 100, and on a 100 MB identity transform 1,840-1,956 MB +// against 1,360-1,400, for the same wall time and 5-10% more CPU. Below it a +// small heap is cheap to hold and the extra CPU is what 200 saves. The size of +// the input is no guide -- DocBook's is small -- so it is the live heap that +// is read. +const cliGCBigHeap = 128 << 20 + // lowerGC applies cliGCPercent unless GOGC is set, so a user who tunes the -// collector keeps their setting. +// collector keeps their setting, and watches the live heap to relax it. func lowerGC() { if os.Getenv("GOGC") == "" { debug.SetGCPercent(cliGCPercent) + watchLiveHeap.Do(armGCSentinel) + } +} + +var watchLiveHeap sync.Once + +type gcSentinel struct{ _ [16]byte } + +// armGCSentinel runs relaxGC after each collection until it has acted. A +// finalizer on an unreachable object runs once the cycle that found it has +// marked, so the live-heap metric it reads is that cycle's. +func armGCSentinel() { + runtime.SetFinalizer(new(gcSentinel), func(*gcSentinel) { + s := []metrics.Sample{{Name: "/gc/heap/live:bytes"}} + metrics.Read(s) + if s[0].Value.Kind() == metrics.KindUint64 && relaxGC(s[0].Value.Uint64()) { + return + } + armGCSentinel() + }) +} + +// relaxGC drops to GOGC=100 when live bytes are past cliGCBigHeap, and +// reports whether it did. +func relaxGC(live uint64) bool { + if live <= cliGCBigHeap { + return false } + debug.SetGCPercent(100) + return true } // paramFlag collects repeated -p name=value flags. From b5aaf06bbd859114f70b222dbb20f9924d99d5cd Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:34:13 -0400 Subject: [PATCH 85/98] xpath: an inline function closure keeps only the variables its body reads --- internal/xpathleaf/locals.go | 12 ++++++++++ xpath/ast.go | 10 +++++++++ xpath/closure_capture_test.go | 41 +++++++++++++++++++++++++++++++++++ xpath/context.go | 32 +++++++++++++++++++++++++++ xpath/funcitem.go | 29 ++++++++++++++++++++++++- xpath/staticvars.go | 23 ++++++++++++++++++++ 6 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 xpath/closure_capture_test.go diff --git a/internal/xpathleaf/locals.go b/internal/xpathleaf/locals.go index 112ac438..4aec00f3 100644 --- a/internal/xpathleaf/locals.go +++ b/internal/xpathleaf/locals.go @@ -108,6 +108,18 @@ func (l *Locals) Frozen() *Locals { vals: append([]xdm.Sequence(nil), l.vals[l.base:]...)} } +// FrozenOnly is Frozen keeping only the bindings keep accepts. +func (l *Locals) FrozenOnly(keep func(uri, local string) bool) *Locals { + f := &Locals{} + for i := l.base; i < len(l.names); i++ { + if n := l.names[i]; keep(n.uri, n.local) { + f.names = append(f.names, n) + f.vals = append(f.vals, l.vals[i]) + } + } + return f +} + // Package xpath sets these in its init; ctx and base are *xpath.Context. // WithLocals returns a child scope of ctx resolving names through l. // WithBindingsOf returns a copy of ctx whose variable bindings are base's. diff --git a/xpath/ast.go b/xpath/ast.go index 38af2182..f874db81 100644 --- a/xpath/ast.go +++ b/xpath/ast.go @@ -315,6 +315,16 @@ type InlineFunctionExpr struct { // Result is the declared return type, or nil if none was written. Result *SequenceType Body Expr + + // needs is what closureNeeds found, computed on first evaluation. + needs atomic.Pointer[closureVars] +} + +// closureVars is the variables a closure's body reads, or all when it must +// capture every one. +type closureVars struct { + names []xdm.QName + all bool } // InlineParam is one parameter of an inline function. diff --git a/xpath/closure_capture_test.go b/xpath/closure_capture_test.go new file mode 100644 index 00000000..127e2f31 --- /dev/null +++ b/xpath/closure_capture_test.go @@ -0,0 +1,41 @@ +package xpath + +import ( + "runtime" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// TestInlineFunctionCapturesOnlyWhatItReads: a closure whose body does not +// mention $big must not keep it alive, and one that does must still see it. +func TestInlineFunctionCapturesOnlyWhatItReads(t *testing.T) { + heldAfterGC := func(src string) (uint64, xdm.Sequence) { + c, err := CompileWith(src, CompileOptions{Version: XPath31}) + if err != nil { + t.Fatal(err) + } + out, err := c.Eval(NewContext(nil, Builtins())) + if err != nil { + t.Fatal(err) + } + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m.HeapAlloc, out + } + var before runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + heldIgnoring, f1 := heldAfterGC(`let $big := (1 to 300000) ! string(.) return function() { 1 }`) + if grown := int64(heldIgnoring) - int64(before.HeapAlloc); grown > 2<<20 { + t.Fatalf("closure ignoring $big retains %d bytes", grown) + } + runtime.KeepAlive(f1) + _, f2 := heldAfterGC(`let $big := (1 to 3) ! string(.), $k := 2 return function($i) { $big[$i] || $k }`) + call := f2[0].(*xdm.FunctionItem) + got, err := call.Invoke(NewContext(nil, Builtins()), []xdm.Sequence{xdm.One(xdm.NewInteger(2))}) + if err != nil || renderSeq(got) != "22" { + t.Fatalf("closure over $big and $k gave %v, %v", got, err) + } +} diff --git a/xpath/context.go b/xpath/context.go index 32220630..96d85882 100644 --- a/xpath/context.go +++ b/xpath/context.go @@ -985,6 +985,38 @@ func frozenLocals(c *Context) *Context { return &n } +// capturedFor is frozenLocals for a closure that reads only the variables keep +// accepts: a scope that binds one name and is not accepted is left out of the +// chain, and a host stack of locals is cut down to the accepted names. Scopes +// binding several names, lazily or by the Vars map stay whole. +func capturedFor(c *Context, keep func(uri, local string) bool, leaf bool) *Context { + if c == nil { + return nil + } + if b := c.bind; b != nil && b.locals != nil { + n := *c + n.bind = &varBinding{locals: b.locals.FrozenOnly(keep)} + return &n + } + p := capturedFor(c.Parent, keep, false) + if b := c.bind; b != nil && b.names == nil && b.lazy == nil && b.local != "" && + len(c.Vars) == 0 && !keep(b.uri, b.local) { + if !leaf { + return p + } + // The innermost scope also carries the environment the body runs in. + n := *c + n.bind, n.Parent = nil, p + return &n + } + if p == c.Parent { + return c + } + n := *c + n.Parent = p + return &n +} + // varKey is a variable's expanded name as a map key that needs no Clark // string built for a lookup. type varKey struct{ uri, local string } diff --git a/xpath/funcitem.go b/xpath/funcitem.go index 0309b552..460c5502 100644 --- a/xpath/funcitem.go +++ b/xpath/funcitem.go @@ -310,7 +310,34 @@ func (e *InlineFunctionExpr) Eval(ctx *Context) (xdm.Sequence, error) { // has no context item, position or size, so "." inside one is XPDY0002 // even where the expression that wrote it had a focus. Variables are // captured; the focus is deliberately dropped. - noFocus := *frozenLocals(ctx) + // + // Only the variables the body reads are kept, so a closure does not + // retain a large value it never mentions. Any binding in a namespace is + // kept: a host's hidden variables, which a function called from the body + // may read, are named there. + cv := e.needs.Load() + if cv == nil { + names, ok := closureNeeds(e) + cv = &closureVars{names: names, all: !ok} + e.needs.Store(cv) + } + var base *Context + if cv.all { + base = frozenLocals(ctx) + } else { + base = capturedFor(ctx, func(uri, local string) bool { + if uri != "" { + return true + } + for _, q := range cv.names { + if q.URI == "" && q.Local == local { + return true + } + } + return false + }, true) + } + noFocus := *base noFocus.Item, noFocus.Position, noFocus.Size = nil, 0, 0 captured := &noFocus item := native(&xdm.FunctionItem{ diff --git a/xpath/staticvars.go b/xpath/staticvars.go index f478c83d..7c587892 100644 --- a/xpath/staticvars.go +++ b/xpath/staticvars.go @@ -153,5 +153,28 @@ func walkFreeVars(e Expr, in bound, out *[]xdm.QName, seen map[string]bool) { case *StringConcat: walkFreeVars(v.Left, in, out, seen) walkFreeVars(v.Right, in, out, seen) + case *dynamicQName: + walkFreeVars(v.arg, in, out, seen) + case *Literal, *ContextItem, *NamedFunctionRef, *ArgumentPlaceholder: + default: + // A node this walk does not know may read any variable. + seen[unknownExpr] = true } } + +// unknownExpr is the key walkFreeVars sets in seen when it meets a node it +// cannot analyse, which closureNeeds reads as "capture everything". +const unknownExpr = "\x00unknown" + +// closureNeeds is the variables an inline function's body reads from the +// scope it was written in, or ok=false when the body holds a node the walk +// does not know and the closure must capture the whole scope. +func closureNeeds(e *InlineFunctionExpr) (names []xdm.QName, ok bool) { + seen := map[string]bool{} + inner := bound(nil) + for _, p := range e.Params { + inner = append(inner, p.Name) + } + walkFreeVars(e.Body, inner, &names, seen) + return names, !seen[unknownExpr] +} From 6293e988c57f8b30dd3e32fad650aede5867235c Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:35:09 -0400 Subject: [PATCH 86/98] xdm: index DTD attribute defaults and types by element name --- xdm/dtd_attnames_test.go | 5 +-- xdm/dtd_defaults.go | 67 +++++++++++++++++++++++++++++++--------- xdm/dtd_defaults_test.go | 42 +++++++++++++++++++++++++ xdm/parse.go | 15 +++++++-- xdm/wellformed.go | 8 ++--- 5 files changed, 113 insertions(+), 24 deletions(-) diff --git a/xdm/dtd_attnames_test.go b/xdm/dtd_attnames_test.go index e2fd0eb9..96841dfb 100644 --- a/xdm/dtd_attnames_test.go +++ b/xdm/dtd_attnames_test.go @@ -45,9 +45,10 @@ func TestDTDAttributeTypingDoesNotBuildNames(t *testing.T) { } el := NewNode(KindElement, QName{Prefix: "schema", Local: "element-with-a-long-name-x"}, "") el.AppendAttr(QName{Prefix: "prefix", Local: "identifier-with-a-long-name"}, "a") + ix := indexDecls(len(types), func(i int) string { return types[i].element }) if n := testing.AllocsPerRun(100, func() { - normalizeAttTokens(tok, types) - applyAttTypes(el, types) + normalizeAttTokens(tok, types, ix) + applyAttTypes(el, types, ix) }); n != 0 { t.Errorf("typing an element against 40 declarations allocated %.0f times, want 0", n) } diff --git a/xdm/dtd_defaults.go b/xdm/dtd_defaults.go index 722c328f..e46df8c9 100644 --- a/xdm/dtd_defaults.go +++ b/xdm/dtd_defaults.go @@ -34,6 +34,49 @@ type attDefault struct { entities map[string]int } +// declIndex maps the element name an ATTLIST declaration was written with to +// the positions of its declarations, in order. Every element is matched +// against the declarations, so scanning them all cost a document of n elements +// and m declarations n*m. +type declIndex map[string][]int + +func indexDecls(n int, element func(i int) string) declIndex { + ix := declIndex{} + for i := range n { + e := element(i) + ix[e] = append(ix[e], i) + } + return ix +} + +// at returns the positions of the declarations that apply to an element +// written prefix:local: those naming it as written and those naming its local +// name alone, in declaration order. +func (ix declIndex) at(prefix, local string) []int { + byLocal := ix[local] + if prefix == "" { + return byLocal + } + var buf [64]byte // stays on the stack for a name that fits + b := append(append(append(buf[:0], prefix...), ':'), local...) + lexical := ix[string(b)] + if len(byLocal) == 0 { + return lexical + } + if len(lexical) == 0 { + return byLocal + } + out := make([]int, 0, len(lexical)+len(byLocal)) + for len(lexical) > 0 && len(byLocal) > 0 { + if lexical[0] < byLocal[0] { + out, lexical = append(out, lexical[0]), lexical[1:] + } else { + out, byLocal = append(out, byLocal[0]), byLocal[1:] + } + } + return append(append(out, lexical...), byLocal...) +} + // parseAttListDefaults extracts the defaulted attributes from a DOCTYPE // internal subset. // @@ -193,16 +236,14 @@ func unquote(s string) string { // that was omitted, it does not override one that was given. Even a #FIXED // default only constrains what may be written, and validating that constraint // is DTD validation, which this parser does not do. -func applyAttDefaults(t xml.StartElement, defs []attDefault) xml.StartElement { +func applyAttDefaults(t xml.StartElement, defs []attDefault, ix declIndex) xml.StartElement { var add []xml.Attr - for _, d := range defs { - // A DTD is not namespace-aware: an ATTLIST names the element as - // written, prefix included. Matching the local name alone missed - // "p:doc", and with it a defaulted xmlns:p the element's own name - // needs. The local-name match stays, as applyAttTypes has it. - if d.element != attrLexical(t.Name) && d.element != t.Name.Local { - continue - } + // A DTD is not namespace-aware: an ATTLIST names the element as + // written, prefix included. Matching the local name alone missed + // "p:doc", and with it a defaulted xmlns:p the element's own name + // needs. The local-name match stays, as applyAttTypes has it. + for _, i := range ix.at(t.Name.Space, t.Name.Local) { + d := defs[i] if hasAttrNamed(t.Attr, d.name) { continue } @@ -274,12 +315,10 @@ func lexicalIs(prefix, local, s string) bool { // Recording the annotation here puts a DTD-validated document on the same // footing as a schema-validated one, so fn:id finds the same attributes in // both. -func applyAttTypes(el *Node, types []attDeclaredType) { +func applyAttTypes(el *Node, types []attDeclaredType, ix declIndex) { en := el.Name() - for _, t := range types { - if !lexicalIs(en.Prefix, en.Local, t.element) && t.element != en.Local { - continue - } + for _, i := range ix.at(en.Prefix, en.Local) { + t := types[i] for a := range el.Attrs() { if an := a.Name(); !lexicalIs(an.Prefix, an.Local, t.attr) && an.Local != t.attr { continue diff --git a/xdm/dtd_defaults_test.go b/xdm/dtd_defaults_test.go index 0882b353..f32b0748 100644 --- a/xdm/dtd_defaults_test.go +++ b/xdm/dtd_defaults_test.go @@ -277,3 +277,45 @@ func TestAttListDefaultValuesAreShared(t *testing.T) { } runtime.KeepAlive(tree) } + +// Each element was matched against every ATTLIST declaration, building the +// element's lexical name for each: 1.4 MB of declarations and elements took +// 11.8 s. The declarations are now indexed by element name, so the cost per +// element does not depend on how many others are declared. Measured: 5 MB +// for 2,000 declarations and 2,000 prefixed elements, against 188 MB scanning +// them all. +func TestAttListDefaultsAreIndexedByElement(t *testing.T) { + const n = 2000 + var dtd, doc strings.Builder + for i := 0; i < n; i++ { + fmt.Fprintf(&dtd, ``, i, i) + } + doc.WriteString("`) + for i := 0; i < n; i++ { + fmt.Fprintf(&doc, ``, i) + } + doc.WriteString("") + var tree *Tree + var err error + b := allocated(func() { tree, err = ParseString(doc.String(), ParseOptions{AllowDOCTYPE: true}) }) + if err != nil { + t.Fatal(err) + } + if b > 24<<20 { + t.Errorf("parse allocated %d MB, want under 24", b>>20) + } else { + t.Logf("parse allocated %d KB", b>>10) + } + i := 0 + for r := range tree.Root.Children() { + for e := range r.Children() { + if got := e.Attr("", "d"); got == nil || got.Value() != fmt.Sprintf("v%d", i) { + t.Fatalf("element %d: default d = %v", i, got) + } + i++ + } + } + if i != n { + t.Errorf("%d elements, want %d", i, n) + } +} diff --git a/xdm/parse.go b/xdm/parse.go index 25427745..54edbb9e 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -322,6 +322,9 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { // the common case is none at all. var attDefaults []attDefault var attTypes []attDeclaredType + // The declarations by element name, built when the first element needs + // them, after the DOCTYPE has settled both lists. + var defIdx, typeIdx declIndex // Element names whose DTD content model is element-only. Whitespace-only // text in such an element is ignorable (XML §2.10) and is stripped // regardless of what the stylesheet declares (XSLT 2.0 §4.4). @@ -394,10 +397,16 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { } } if len(attDefaults) > 0 { - t = applyAttDefaults(t, attDefaults) + if defIdx == nil { + defIdx = indexDecls(len(attDefaults), func(i int) string { return attDefaults[i].element }) + } + t = applyAttDefaults(t, attDefaults, defIdx) } if len(attTypes) > 0 { - t = normalizeAttTokens(t, attTypes) + if typeIdx == nil { + typeIdx = indexDecls(len(attTypes), func(i int) string { return attTypes[i].element }) + } + t = normalizeAttTokens(t, attTypes, typeIdx) } if err := validateStartElement(t, &ns, dec.IsVersion11()); err != nil { return nil, err @@ -423,7 +432,7 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { } } if len(attTypes) > 0 { - applyAttTypes(el, attTypes) + applyAttTypes(el, attTypes, typeIdx) } // Attributes and namespaces are nodes too, and a document made // of elements carrying many attributes allocates most of its diff --git a/xdm/wellformed.go b/xdm/wellformed.go index 8230ae11..6841e6f1 100644 --- a/xdm/wellformed.go +++ b/xdm/wellformed.go @@ -364,12 +364,10 @@ func requireQName(n xml.Name) error { // attributes to the raw tag, ahead of validateStartElement. Namespaces in // XML §3 binds a prefix to the attribute's *normalized* value, so xmlns:b // declared NMTOKEN and written " urn:x " binds urn:x — and p:a, b:a collide. -func normalizeAttTokens(t xml.StartElement, types []attDeclaredType) xml.StartElement { +func normalizeAttTokens(t xml.StartElement, types []attDeclaredType, ix declIndex) xml.StartElement { var attrs []xml.Attr - for _, d := range types { - if !lexicalIs(t.Name.Space, t.Name.Local, d.element) && d.element != t.Name.Local { - continue - } + for _, i := range ix.at(t.Name.Space, t.Name.Local) { + d := types[i] for i, a := range t.Attr { if !lexicalIs(a.Name.Space, a.Name.Local, d.attr) && a.Name.Local != d.attr { continue From 996b7db9ff2974110e36f27d7bcdcd99596b312a Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:38:31 -0400 Subject: [PATCH 87/98] xsd: extend a key scope's entries instead of rebuilding them per level --- xsd/identity.go | 31 +++++++++++++--- xsd/identity_entries_test.go | 70 ++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 xsd/identity_entries_test.go diff --git a/xsd/identity.go b/xsd/identity.go index 436e3408..dcf48790 100644 --- a/xsd/identity.go +++ b/xsd/identity.go @@ -67,6 +67,12 @@ type nodeTable struct { // the document makes unique. It oscillated with the count: wrong at // three siblings and five, right at two and four. ambiguous map[string]bool + + // dups records that targets holds two nodes with one key sequence, which + // buildNodeTable reports again in every enclosing scope. While it is + // false, and ambiguous is empty, entries is exactly targets inverted and + // an enclosing scope may extend it as it is. + dups bool } // mergeEntry folds one sibling's entry into this table. @@ -272,6 +278,7 @@ func mergeTables(children []icTables) icTables { // A sequence already ambiguous in the child stays ambiguous // here: the ancestor's scope contains that child's whole // subtree, so both occurrences are inside it. + existing.dups = existing.dups || tbl.dups for k := range tbl.ambiguous { if existing.ambiguous == nil { existing.ambiguous = map[string]bool{} @@ -303,10 +310,7 @@ func mergeTables(children []icTables) icTables { // buildNodeTable evaluates a key or unique constraint over an element's // subtree. func (v *validator) buildNodeTable(el *xdm.Node, ic *IdentityConstraint, below *nodeTable) *nodeTable { - tbl := &nodeTable{ - entries: map[string]*xdm.Node{}, - targets: map[*xdm.Node]string{}, - } + tbl := &nodeTable{} // A per-element check in validateElement does not reach this loop: one // constraint at the top of a deeply recursive document selects the whole @@ -365,6 +369,25 @@ func (v *validator) buildNodeTable(el *xdm.Node, ic *IdentityConstraint, below * // carrying it would leave a table holding an order that covers // only part of its targets, for a later reader to trust. tbl.targets = below.targets + if !below.dups && len(below.ambiguous) == 0 { + // No sequence is held twice, so the subtree's entries are its + // targets inverted and there is nothing to report: they are + // extended in place. Rebuilding them, and sorting the targets + // to do it, at every level of a recursive document cost + // O(targets x depth), 730 MB for a 258 KB document. + tbl.entries = below.entries + } else { + tbl.dups = true + tbl.entries = map[string]*xdm.Node{} + } + } + if tbl.entries == nil { + tbl.entries = map[string]*xdm.Node{} + } + if tbl.targets == nil { + tbl.targets = map[*xdm.Node]string{} + } + if tbl.dups { // Walked in document order, not map order, so a duplicate is // reported on its later occurrence and several are reported in // the same order on every run. diff --git a/xsd/identity_entries_test.go b/xsd/identity_entries_test.go new file mode 100644 index 00000000..923567ce --- /dev/null +++ b/xsd/identity_entries_test.go @@ -0,0 +1,70 @@ +package xsd + +import ( + "runtime" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// A key scope rebuilt its entries, and sorted its targets to do it, from the +// whole subtree at every level of a recursive document: 960 levels with two +// leaves each allocated 84 MB, and a 258 KB document 730 MB. With no sequence +// held twice the scope extends the entries it was handed, so the cost per +// level is the gap to the next scope. Measured: 1 MB here. +func TestKeyScopeDoesNotRebuildEntriesPerLevel(t *testing.T) { + st, err := xdm.ParseString(identityBenchSchema(), xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + s, err := Load(st.Root, "", Options{}) + if err != nil { + t.Fatal(err) + } + tr, err := xdm.ParseString(identityBenchDoc(960, 2), xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var verr error + var m0, m1 runtime.MemStats + runtime.ReadMemStats(&m0) + verr = s.Validate(tr.Root, ValidateOptions{}) + runtime.ReadMemStats(&m1) + if verr != nil { + t.Fatal(verr) + } + if got := m1.TotalAlloc - m0.TotalAlloc; got > 16<<20 { + t.Errorf("validation allocated %d MB, want under 16", got>>20) + } +} + +// A sequence two sibling scopes both hold is a duplicate in the scope that +// contains them, and is reported again by every scope above, so the fast path +// above must not apply once a table holds one. +func TestKeyScopeSiblingDuplicateStillReportedPerScope(t *testing.T) { + const sch = ` + + + + + + + ` + st, err := xdm.ParseString(sch, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + s, err := Load(st.Root, "", Options{}) + if err != nil { + t.Fatal(err) + } + tr, _ := xdm.ParseString("", xdm.ParseOptions{}) + err = s.Validate(tr.Root, ValidateOptions{MaxErrors: -1}) + if err == nil { + t.Fatal("duplicate key accepted") + } + if n := strings.Count(err.Error(), "duplicate key sequence"); n != 2 { + t.Errorf("%d duplicate reports, want 2 (the scope holding both, and its parent): %v", n, err) + } +} From de0ac46b3765465025a125ac94872aa96f6edaef Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 00:45:12 -0400 Subject: [PATCH 88/98] docs: record the third memory audit's fixes Changelog rows, the nineteenth security pass, the deep-equal bound, the MaxDigits and CLI GC notes, the open items in known-gaps, and regenerated test counts. --- CHANGELOG.md | 51 ++++++++++++++++++++++++++++++++++++++-- README.md | 4 ++-- docs/benchmark.md | 3 ++- docs/conformance-gaps.md | 2 +- docs/known-gaps.md | 34 +++++++++++++++++++++++++++ docs/options.md | 1 + docs/security.md | 10 +++++++- docs/stats.md | 2 +- docs/testing.md | 2 +- docs/todo.md | 2 +- 10 files changed, 101 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2794a46a..b9fa0b54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ Three new exported functions make this a minor release. | `xdm.Node.AddNamespaces` | Sets several namespace bindings on an element in one call. | [`b42f04a`][b42f04a] | | `xdm.ArrayItem.AppendMembers` | Appends an array's members to a slice without an intermediate copy. | [`59ac569`][59ac569] | | `xpath.CompileBounded`, `xpath.MaxRegexpProgram` | `regexp.Compile` that refuses a program over 100,000 instructions; used by XPath and XSD patterns. | [`c1c2bb9`][c1c2bb9] | +| `xpath.Env.MaxDigits`, `xpath.MaxDigits` | Bounds the digits of an `xs:integer` or `xs:decimal` cast or arithmetic result; 0 is 10^6, negative is none. | [`8577906`][8577906] | +| `xdm.AppendIntegers`, `xpath.Context.WithVars` | Appends a run of integers allocated a chunk at a time; binds several variables in one scope. | [`30c7c07`][30c7c07], [`5951f24`][5951f24] | | `xpath.Context.NodesRefused`, `ItemsRefused` | Whether the node budget was crossed, and whether a running item total is past the item bound, without charging either. | [`4e8f0a5`][4e8f0a5], [`21995a6`][21995a6] | ### Changed @@ -26,7 +28,9 @@ Three new exported functions make this a minor release. |---|---|---| | `xsd` and `dtd` error messages clip document-supplied strings to 1 KB plus "..." | Nothing, unless you match messages longer than 1 KB. Message wording is not part of the contract. | [`20958aa`][20958aa], [`bd865e4`][bd865e4] | | json and adaptive serialization can leave partial output when an error is raised mid-document | Treat the writer's content as invalid on error, as the XML methods already required. | [`599b2f9`][599b2f9] | -| New refusals: maps and arrays nested past 1,000 levels in json/adaptive output (XPDY0130) or 500 in `deep-equal` and `map:find` (XPDY0001); regexes over 100,000 instructions (FORX0002, or a pattern error in XSD); picture widths over 65,536 (FOFD1340); `xsl:key` and `group-by` keys past 5,000,000 retained (XPDY0130) | Nothing for real inputs: the largest W3C suite regex is 4,313 instructions. All wrap `xdm.ErrResourceLimit`. | [`702c6e6`][702c6e6], [`c1c2bb9`][c1c2bb9], [`21995a6`][21995a6] | +| New refusals: maps and arrays nested past 1,000 levels in json/adaptive output (XPDY0130), `deep-equal` and `map:find` (XPDY0001), with their own counter, apart from function recursion; regexes over 100,000 instructions (FORX0002, or a pattern error in XSD); picture widths over 65,536 (FOFD1340); `xsl:key` and `group-by` keys past 5,000,000 retained (XPDY0130); integers over 10^6 digits (FOCA0003, FOAR0002); sequences built by `,` or `!`, `xsl:next-iteration` values, `unparsed-text-lines` lines, `parse-json`/`json-doc` values and stored accumulator items past `MaxItems` (XPDY0130); `xsl:copy-of` of a document past `MaxNodes` (XPDY0130) | Nothing for real inputs: the largest W3C suite regex is 4,313 instructions. All wrap `xdm.ErrResourceLimit`. | [`702c6e6`][702c6e6], [`014a862`][014a862], [`c1c2bb9`][c1c2bb9], [`21995a6`][21995a6], [`8577906`][8577906], [`dcb49b2`][dcb49b2], [`c73a06b`][c73a06b], [`a3d8ef6`][a3d8ef6], [`40cd52c`][40cd52c], [`b35ec14`][b35ec14] | +| The CLI drops its collector from `GOGC=200` to 100 once the live heap passes 128 MB | Nothing; set `GOGC` to keep a fixed value. Peak RSS −20% on DocBook's largest table, about 12% more user CPU. | [`f3449f7`][f3449f7] | +| The `FileResolver` no longer evicts to admit a document that does not fit beside its cache | A document refused admission is parsed again on its next request, so across transforms it is not the same tree; within one transform it is. | [`f49ff92`][f49ff92] | | `xdm.ProcessXInclude` returns a parsed tree unchanged when it has no XInclude element | Nothing; a parsed tree is frozen. A built tree is still copied. | [`e139525`][e139525], [`9c3a983`][9c3a983] | ### Fixed @@ -37,6 +41,11 @@ Three new exported functions make this a minor release. | The DTD validator built every element's path, quadratic in depth | A valid 990-deep chain of 4 KB names allocated 2 GB. The path is rendered only for a failure, text unchanged. | [`8dcb07f`][8dcb07f] | | DTD validity messages kept `#FIXED`, enum and ID values whole | 200 mismatches on a 1 MB `#FIXED` value kept 105 MB. `fail` clips to 1 KB. | [`bd865e4`][bd865e4] | | The nested `fn:transform` cache kept compiles for dead resolvers | A fresh resolver per call kept 49 MB on DocBook. Resolvers are weak keys and dead entries are dropped: 0.3 MB. | [`a5b385d`][a5b385d] | +| Sequences built by `,` or `!` were not counted against `MaxItems` | A 70-byte doubling `fold-left` built 134M items against the 5M limit, at 6.3 GB; through recursive `with-param` too. They are charged above 64 items. Also in v2.0.3. | [`dcb49b2`][dcb49b2] | +| RELAX NG reported a valid document invalid | `(a?){13} a{13}` against 13 `` hit the derivative cap, naming a `oneOrMore` the schema lacks; `xmllint` accepts it. Choice chains merge alternative by alternative: N = 12, 13, 18, 30 validate. Also in v2.0.3. | [`44229aa`][44229aa] | +| Serializing a sequence of atomic values was quadratic | `1 to 100000` took 21.6 s and 6.95 GB, joined by repeated concatenation. Each run is joined once: 0.056 s. Also in v2.0.3. | [`a9ad280`][a9ad280] | +| Text lines, JSON values, DTD default values, integer digits and accumulator values grew past the limits | `unparsed-text-lines` kept 5.2 GB, 2 MB of DTD defaults 1.97 GB, a 4M-digit cast 34 GB, a 200 KB accumulator 12.4 GB. Each is charged, bounded, stored once or shared. Also in v2.0.3. | [`c73a06b`][c73a06b], [`be5afc3`][be5afc3], [`8577906`][8577906], [`40cd52c`][40cd52c] | +| `xsl:copy-of` of a document was not counted against `MaxNodes` | It copied 5.7M nodes and exited 0, where shallow-copy is refused. It is charged before copying. | [`b35ec14`][b35ec14] | | Deeply nested maps and arrays crashed the process | `serialize(..., json)` or `deep-equal` on a 400,000-deep array from a 96-byte query overflowed the Go stack. They are refused with a resource-limit error; `xdm.Flatten` is iterative. Also in v2.0.3. | [`702c6e6`][702c6e6] | | A built-in `shallow-copy` rule returned a truncated result past `MaxNodes` with no error | A 1M-element identity copy emptied a third of its elements and exited 0. The transform now fails with XPDY0130 whenever the node budget was crossed. Also in v2.0.3. | [`4e8f0a5`][4e8f0a5] | | Sequential regex repeats compiled to huge programs | `\pL{1000}` repeated 150 times passes Go's 1,000-repeat check: 181 KB of patterns kept 984 MB. Programs over 100,000 instructions are refused in XPath and XSD. | [`c1c2bb9`][c1c2bb9] | @@ -63,7 +72,14 @@ Three new exported functions make this a minor release. | c14n `Bytes` and `Result.String` doubled a buffer from empty; `Serialize` allocated 32 KB a call | Output chunks are joined once and the write buffer is pooled: c14n 3.7× → 2.1× the output, a small result 33.6 → 0.7 KB. | [`f235795`][f235795], [`9496810`][9496810], [`37206b1`][37206b1], [`c501fc4`][c501fc4], [`1f18b1b`][1f18b1b] | | The `xslt` benchmarks read inputs no checkout has, so they always skipped | They run XRechnung and Peppol CEN from `testdata/bench`. | [`beb6072`][beb6072] | | `-xinclude` copied every tree, and included documents twice, even with nothing to include | A parsed tree with no XInclude element is returned as is and inclusions are built into the destination: `parse-100mb` 514 → 0 MB. | [`e139525`][e139525] | -| RELAX NG copied its name memo on every store | Stores land in a pending map published as a snapshot: 1,024 stores 47.5 → under 2 MB. | [`9f0db9c`][9f0db9c] | +| RELAX NG copied its name memo on every store | Stores land in a pending map published as a snapshot, and after 8 locked hits: 1,024 stores 47.5 → under 2 MB, lookups lock-free. | [`9f0db9c`][9f0db9c], [`676ad52`][676ad52] | +| Integer `mod`, `idiv`, negation and comparison used big arithmetic; ranges made 3 allocations an integer | int64 fast paths and chunked ranges: comparison 2 → 0 allocations, `sum(1 to 1e6)` 800k → 4.8k allocations. | [`40085f6`][40085f6], [`30c7c07`][30c7c07] | +| Dynamic calls, FLWOR tuples and window clauses made a scope per variable; closures kept their whole scope | One scope per binding set, and a closure keeps only what its body reads: `$f($i)` over 1M 2,673 → 1,617 MB. | [`5951f24`][5951f24], [`a4f316d`][a4f316d], [`b5aaf06`][b5aaf06] | +| The function result memo kept each key string | It keeps a 128-bit SHA-256 hash: DocBook `table-cals.049` live heap 359 → 172 MB, peak 1,150 → 533 MB. | [`15273c7`][15273c7] | +| `fn:doc` read the file whole and copied it; `replace` copied and over-charged a non-matching input | `fn:doc` parses as it reads (16 MB file 82 → 29 MB); `replace` returns early with no match (64 MB: 256 → 0 MB). | [`a6f2eaa`][a6f2eaa], [`6bab64a`][6bab64a] | +| `xsl:sort`, `xsl:iterate`, `xsl:result-document`, built-in shallow-copy, namespace scopes and `on-empty` allocated per item | Smaller sort values in one array, parameter slices, set lookups, no focus copy without rules, frame walks, a lazy leaf walk. `result-document` is linear. | [`6ea432f`][6ea432f], [`a3d8ef6`][a3d8ef6], [`951f0d9`][951f0d9], [`f650a19`][f650a19], [`4b0b564`][4b0b564], [`469c95e`][469c95e] | +| DTD defaults were scanned per element; XSD key scopes rebuilt their entries per level | Defaults indexed by element (188 → 5 MB); scopes extend in place (960 levels 84 → 1 MB). | [`6293e98`][6293e98], [`996b7db`][996b7db] | +| Large regex programs were cached; `parse-json` objects and `serialize` refusals cost per key and item | Programs over 4,096 instructions are not cached; objects use `MapBuilder` (about 3.3× less); `serialize` stops at the first refusal. | [`331bc96`][331bc96], [`98811f6`][98811f6], [`6317d49`][6317d49] | | DTD validation built a state-set map, a name slice and an attribute map per element | Small sets dedup by scan, the slice is reused, no ATTLIST skips the map: 12,062 → 8,062 allocations for 6,001 elements. | [`5dc120c`][5dc120c] | | `xsdnet.HTTPResolver` built a new transport per fetch | One guarded transport is shared, so connections are reused; the SSRF guard is unchanged. | [`da05b16`][da05b16] | | Typed copies pinned dropped schemas; `array:append` copied twice; line index, interning key and memo key kept slack | Weak typing-name slots, one copy, clipped index, reused key buffer, oversized memo buffer dropped. | [`967dfb3`][967dfb3], [`59ac569`][59ac569], [`3b71501`][3b71501], [`69414ae`][69414ae], [`887d5d3`][887d5d3] | @@ -1742,3 +1758,34 @@ here so every entry in this file sits under a release. [3b71501]: https://github.com/knroy/go-xml/commit/3b71501 [69414ae]: https://github.com/knroy/go-xml/commit/69414ae [887d5d3]: https://github.com/knroy/go-xml/commit/887d5d3 +[8577906]: https://github.com/knroy/go-xml/commit/8577906 +[30c7c07]: https://github.com/knroy/go-xml/commit/30c7c07 +[5951f24]: https://github.com/knroy/go-xml/commit/5951f24 +[014a862]: https://github.com/knroy/go-xml/commit/014a862 +[dcb49b2]: https://github.com/knroy/go-xml/commit/dcb49b2 +[c73a06b]: https://github.com/knroy/go-xml/commit/c73a06b +[a3d8ef6]: https://github.com/knroy/go-xml/commit/a3d8ef6 +[40cd52c]: https://github.com/knroy/go-xml/commit/40cd52c +[b35ec14]: https://github.com/knroy/go-xml/commit/b35ec14 +[f3449f7]: https://github.com/knroy/go-xml/commit/f3449f7 +[f49ff92]: https://github.com/knroy/go-xml/commit/f49ff92 +[44229aa]: https://github.com/knroy/go-xml/commit/44229aa +[a9ad280]: https://github.com/knroy/go-xml/commit/a9ad280 +[be5afc3]: https://github.com/knroy/go-xml/commit/be5afc3 +[676ad52]: https://github.com/knroy/go-xml/commit/676ad52 +[40085f6]: https://github.com/knroy/go-xml/commit/40085f6 +[a4f316d]: https://github.com/knroy/go-xml/commit/a4f316d +[b5aaf06]: https://github.com/knroy/go-xml/commit/b5aaf06 +[15273c7]: https://github.com/knroy/go-xml/commit/15273c7 +[a6f2eaa]: https://github.com/knroy/go-xml/commit/a6f2eaa +[6bab64a]: https://github.com/knroy/go-xml/commit/6bab64a +[6ea432f]: https://github.com/knroy/go-xml/commit/6ea432f +[951f0d9]: https://github.com/knroy/go-xml/commit/951f0d9 +[f650a19]: https://github.com/knroy/go-xml/commit/f650a19 +[4b0b564]: https://github.com/knroy/go-xml/commit/4b0b564 +[469c95e]: https://github.com/knroy/go-xml/commit/469c95e +[6293e98]: https://github.com/knroy/go-xml/commit/6293e98 +[996b7db]: https://github.com/knroy/go-xml/commit/996b7db +[331bc96]: https://github.com/knroy/go-xml/commit/331bc96 +[98811f6]: https://github.com/knroy/go-xml/commit/98811f6 +[6317d49]: https://github.com/knroy/go-xml/commit/6317d49 diff --git a/README.md b/README.md index a0e5e4ac..a43a21ae 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ and maintains it as a project of his own. | **XSD 1.1** | 99.98% instance (26,217 of 26,222); **99.97%** schema-validity (15,350 of 15,354); opt-in via `Version11` | | **RELAX NG** | 100.00% of James Clark's spectest (965 of 965 assertions); XML and compact syntax, from Go and from `go-xml validate -rng` | | **DTD** | content models, attribute defaults, enumerations, `ID`/`IDREF`; external subset, parameter entities across both subsets, conditional sections — via `dtd.Load` with a caller-supplied resolver, nothing fetched by default | -| **Tests** | 2,907 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | +| **Tests** | 2,941 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | | **Production schemas** | UBL 2.1, UN/CEFACT CII, Factur-X/ZUGFeRD, Peppol BIS 3.0 — 88 schemas load, instances validate clean | | **API** | 1.2; the exported surface is stable and additive over 1.1, and a breaking change means 2.0 with a new module path | @@ -1388,7 +1388,7 @@ back, is in [docs/testing.md](docs/testing.md). | method | what it catches | what it misses | |---|---|---| -| **Unit tests** (2,907 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** (2,941 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | | **Spec inventories** | features absent entirely | features present but behaving wrongly | | **Saxon differential** | subtle behavioural divergence on real stylesheets | constructs the corpora do not use | | **W3C QT3 suite** | systematic conformance across 22,054 XPath and 30,517 XQuery cases | XSLT (it is an XPath suite) | diff --git a/docs/benchmark.md b/docs/benchmark.md index 930d0690..83605b60 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -472,7 +472,8 @@ Peak RSS in cold runs (median / maximum over items): For transforms v1 uses a fifth to two thirds of a JVM's memory, and v2 an eighth to a quarter. The CLI runs its collector at `GOGC=200` unless `GOGC` -is set, which trades some peak memory for CPU. For large documents v1's tree +is set, which trades some peak memory for CPU, and drops to `GOGC=100` once +the live heap passes 128 MB. For large documents v1's tree cost about 23 bytes of RSS per byte of input, 1.6× libxml2's; v2's costs under 4, about a quarter of libxml2's. Only `encoding/xml`, which builds no tree, uses less. diff --git a/docs/conformance-gaps.md b/docs/conformance-gaps.md index 2bde3419..8b70b605 100644 --- a/docs/conformance-gaps.md +++ b/docs/conformance-gaps.md @@ -49,7 +49,7 @@ W3C disagreements: 0 + 0 + 0 + 1 + 8 + 23 + 30 + 31 + 0 = 93. Measured 2026-09-1 -The unit-test suite is 2,907 tests. +The unit-test suite is 2,941 tests. The last two rows are not W3C suites but real-world corpora — DocBook xslTNG's diff --git a/docs/known-gaps.md b/docs/known-gaps.md index 551f0099..2efbb3d8 100644 --- a/docs/known-gaps.md +++ b/docs/known-gaps.md @@ -105,6 +105,40 @@ tables being copied per scope and target nodes being walked again. Found by an external audit of v2.0.0; the figures are the `BenchmarkIdentity*` benchmarks in `xsd/`. +### Memory: what the third memory audit left open (resource limits, performance) + +Found by the third memory audit (2026-11-01) and not fixed, each with the reason: + +- **No bound on the total across many `fn:doc` calls.** The per-document + `MaxBytes` holds, but 40 distinct 500k-element documents were held at 846 MB + live. A correct total needs a per-evaluation set of loaded URIs (the + `FileResolver` cache is shared, so charging on a miss undercounts), state + carried into nested `fn:transform`, and every reader covered. Lower + `FileResolver.MaxBytes` or the allowed roots meanwhile. +- **`Env.MaxDigits` covers casts, constructors and binary arithmetic, not + `CastAtomic`, `fn:sum` or `fn:avg`**, which take no context. +- **RELAX NG's derivative cap still refuses some valid documents.** The choice + merge fixed `(a?){n} a{n}` up to about n = 30. At n = 50 and 100 the cap is + reached again, though in 20–300 ms rather than 19.9 s. Hash-consing + derivatives would close it. +- **`xsl:number level="any"` is quadratic in CPU** (40k items 44 s). A cached + prefix count is not valid, since `count` and `from` patterns may use + `current()` or per-call variables. +- **`fold-left` appending one item a step is quadratic in allocation** + (40k items 12.4 GB, all garbage). Appending in place is unsound while other + references may share the prefix. +- **Identity templates allocate about 3.5 KB a node of context copies** + (`withSel`, `hostWithCurrent`, the leaf-hook copy), and built-in shallow-copy + copies a document-level result twice; removing that copy changes an error + case into output. +- **Accumulators keep two per-node maps.** Node-indexed slices need an ordinal + `xdm.Node` does not export. +- **XQuery inline functions capture their whole scope.** The XPath and XSLT + closure (`InlineFunctionExpr`) keeps only what its body reads; + `xquery/inline.go` has its own implementation. +- **A regex program over 4,096 instructions is compiled on every call**, + because it is not cached; the largest QT3 pattern is 4,059. + ### `fn:replace` charges a backtracking or backreference result after building it (XPath, resource limits) The RE2 path of `fn:replace` charges `len(input) + matches × len(replacement)` diff --git a/docs/options.md b/docs/options.md index cff27fb9..3f0880f3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -688,6 +688,7 @@ functions or `WithEnv`): | `RegexVersion`, `LibraryVersion` | `Version` | Raise the regular-expression dialect and the function library above the expression's version, without admitting new syntax. | | `MaxDepth` | `int` | Recursion bound. Zero is `xpath.MaxDepth` (500). | | `MaxItems` | `int` | Items an evaluation may materialise. Zero is `xpath.MaxItems` (5,000,000); negative is no bound; a nested evaluation adopts the caller's bound and cannot raise it. | +| `MaxDigits` | `int` | Digits of an `xs:integer` or `xs:decimal` a cast or arithmetic may produce. Zero is `xpath.MaxDigits` (10^6), `FOCA0003` or `FOAR0002` past it; negative is no bound. | | `QualifyVar`, `MissingVar` | funcs | Host hooks for variable references (XSLT packages, XQuery cycles). | | `MapDuplicateCode` | `string` | Error code for a duplicate key in a map constructor; `XQDY0137` by default. | diff --git a/docs/security.md b/docs/security.md index b8c9ec87..c71a9bbf 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1865,11 +1865,19 @@ Each line names the **direction** of the defect, because that is what decides who was exposed: a *false accept* let an invalid input through, a *false reject* refused a legal one, and *cost* produced the right answer too slowly. +**Nineteenth pass — a third memory audit, 2026-11-01.** The findings and figures are in the audit's own write-up; the retention and allocation fixes are in CHANGELOG under *Changed — memory*. + +- **Sequences built by `,` and `!` were not counted against `MaxItems`** — availability, driven by a query or stylesheet. `fold-left(1 to 26,(1,2),function($s,$i){($s,$s)})` (70 bytes) built 134M items against the 5M limit at 6.3 GB, and the same doubling through a recursive `xsl:with-param` reached 4.6 GB on v2.0.3. `SequenceExpr` and `!` charge the copies beyond their widest operand above 64 items, and `xsl:next-iteration` values are checked against the bound. Pinned by `TestSequenceCopiesAreCharged`. +- **RELAX NG rejected a valid document** — correctness. `(a?){13} a{13}` against 13 `` crossed the 100,000-node derivative cap, and the message named a `oneOrMore` the schema does not have; `xmllint` accepts it. Choice chains now merge alternative by alternative, under a 1,024-pair budget per merge so a hostile schema stays bounded. Pinned by `TestOptionalsBeforeRequiredStayValid`. Larger N still reaches the cap (docs/known-gaps.md). +- **Serializing atomic values was quadratic** — cost, driven by result size. `xslt/seqnorm.go` joined adjacent atomics by repeated concatenation, so `1 to 100000` took 21.6 s and 6.95 GB for 588 KB of output. Each run is joined once. Pinned by `TestJoinAdjacentAtomicsIsLinear`. +- **Several producers were uncharged or unbounded** — cost, driven by document or query data. `unparsed-text-lines`, `parse-json` and `json-doc` built tens of millions of items past `MaxItems` (5.2 GB from a 67 MB file); DTD default values were copied into every element unseen by `MaxNodes` or `MaxBytes` (2 MB retained 1.97 GB); an `xs:integer` had no size bound (a 4M-digit cast allocated 34 GB); a sequence-valued accumulator copied its value into every node (200 KB held 12.4 GB). Lines and JSON values are charged, defaults are stored once, `Env.MaxDigits` (10^6) refuses with FOCA0003 or FOAR0002, and accumulator values share their prefix and are charged. +- **`xsl:copy-of` of a document bypassed `MaxNodes`** — the inconsistency, not data loss: it copied 5.7M nodes and exited 0 where shallow-copy is refused. It is charged before the copy. + **Eighteenth pass — a memory audit, 2026-10-31.** The rest of that audit was retention and allocation with no hostile driver; it is in CHANGELOG under *Changed — memory*. - **`xsd` error messages kept document strings whole** — cost, driven by document data. `fail` and `attrName` quoted names and values in full and the validator keeps up to 100 errors, so 200 foreign attributes in a 1 MB namespace retained 105 MB, and one 15 MB value under a failing rule set retained 1.57 GB (audit 20). Strings and the formatted message are clipped to 1 KB plus "...", and the pattern and list-item errors quote through `truncate`: 0.13 MB. Pinned by `TestValidationMessagesClipDocumentStrings` in `xsd/message_clip_test.go`. - **The DTD validator built every element's path, quadratic in depth** — cost, driven by document data. The path string was rebuilt at each element whether or not anything failed: a *valid* 990-deep chain of 4 KB names allocated 2 GB. It is rendered from the ancestors only when a failure is recorded, and an IDREF keeps its element rather than a path. Its `#FIXED`, enum and ID values are clipped to 1 KB too. Pinned by `TestPathBuiltOnlyForFailures` and `TestValidityMessagesClipLongValues` in `dtd/`. -- **Deeply nested maps and arrays overflowed the Go stack** — availability, driven by a query or stylesheet. `serialize(..., map{"method":"json"})`, the adaptive method, `fn:deep-equal` and `map:find` recursed once per level, so `fold-left(1 to 400000, [], function($a,$i){[$a]})` (96 bytes) ended the process with `fatal error: stack overflow`, which no `recover` can catch; v2.0.3 has it too. Serialization refuses past 1,000 levels (the `parse-json` limit) with XPDY0130, `deep-equal` and `map:find` past the 500-level recursion bound with XPDY0001, and `xdm.Flatten` is iterative. Pinned by `TestDeepNestedValuesAreRefused` in `xpath/deepnest_test.go`. +- **Deeply nested maps and arrays overflowed the Go stack** — availability, driven by a query or stylesheet. `serialize(..., map{"method":"json"})`, the adaptive method, `fn:deep-equal` and `map:find` recursed once per level, so `fold-left(1 to 400000, [], function($a,$i){[$a]})` (96 bytes) ended the process with `fatal error: stack overflow`, which no `recover` can catch; v2.0.3 has it too. Serialization refuses past 1,000 levels (the `parse-json` limit) with XPDY0130, `deep-equal` and `map:find` past 1,000 levels on a counter of their own (XPDY0001), so function recursion does not lower it, and `xdm.Flatten` is iterative. Pinned by `TestDeepNestedValuesAreRefused` in `xpath/deepnest_test.go`. - **A built-in rule returned a truncated result tree as a success** — correctness, driven by document size. The result builder latches a `MaxNodes` refusal, but only instruction bodies checked it, so `` over 1M elements emptied a third of them and exited 0; v2.0.3 has it too. The transform now fails with XPDY0130 whenever the node budget was crossed anywhere, through `xpath.Context.NodesRefused`. Pinned by `TestBuiltInRuleResultTreeOverrunIsRefused`. - **`fn:replace`, picture widths, `xsl:key` and `group-by` grew past their budgets** — cost, driven by document or stylesheet data. `replace` charged its result after building it (12.8 GB for a 500 KB input and a 4 KB replacement); a `format-date` width of `[D01,1000000000]` (52 bytes) allocated 2.8 GB through `strings.Repeat`; `use="1 to 1000"` over 100k nodes built 100M key entries at 3.2 GB peak, since the item counter resets per expression. `replace` charges first on the RE2 path (the other two engines are a known gap), widths over 65,536 are FOFD1340, and keys are totalled per index and grouping against `MaxItems`. - **Caches bounded by entry count held gigabytes** — cost, driven by document data or the host. 1,024 cached regexes of 200 KB, 256 failed 1 MB collation URIs, or 256 large `FileResolver` documents each held hundreds of MB to GB. Keys over 1 KB are not cached, and the document cache is also bounded by 32 MB of source. diff --git a/docs/stats.md b/docs/stats.md index 13b80337..f32651b2 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -23,7 +23,7 @@ claim is only as good as the command behind it. | figure | count | counted by | |---|---:|---| -| Unit tests | 2,907 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | +| Unit tests | 2,941 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Fuzz targets | 12 | `grep -rn '^func Fuzz' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Limit boundary tests | 14 | `grep -hc '^func Test' ./*/limits_boundary*_test.go \| awk '{n += $1} END {print n + 0}'` | diff --git a/docs/testing.md b/docs/testing.md index 28ffbab7..8ff8d51e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,7 +21,7 @@ let something through, and the column that matters is the last one. | layer | count | catches | misses | |---|---:|---|---| -| **Unit tests** | 2,907 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** | 2,941 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | | **Limit boundary tests** | 14 tests | an off-by-one or an overflow at the edge of a configurable limit | a limit nobody added to the inventory | | **Race detector** | same tests | shared state a single-goroutine run never reveals | a data race on a path no test walks | | **W3C conformance suites** | 152,241 cases | systematic divergence from the specification | what the suites do not ask about — see below | diff --git a/docs/todo.md b/docs/todo.md index 62c44c6b..17727f72 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -23,7 +23,7 @@ Current position: | XSLT 3.0 | 99.80% — 11,495 of 11,518 in scope (23 failing); 3 of those need more of the §19.8 streamability analysis | | RELAX NG | 100.00% — 965 of 965 | | Schemas wrongly refused | 7 — 6 on XSD 1.0, 1 on 1.1 | -| Tests | 2,907 `func Test` declarations, clean under `-race` | +| Tests | 2,941 `func Test` declarations, clean under `-race` | Every one of those failures, and why it is still open, is catalogued in [known-gaps.md](known-gaps.md). This file is the forward-looking half — what From b37796b3f23a17fcf6c7e1635625f3c5ff52aaec Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:00:53 -0500 Subject: [PATCH 89/98] xdm: add Node.Index, Node.SubtreeEnd, Tree.Len and Tree.NodeAt --- xdm/index_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++ xdm/record.go | 47 ++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 xdm/index_test.go diff --git a/xdm/index_test.go b/xdm/index_test.go new file mode 100644 index 00000000..40724f2d --- /dev/null +++ b/xdm/index_test.go @@ -0,0 +1,110 @@ +package xdm + +import ( + "bytes" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +// checkIndex walks every node of t, attributes included, in document order and +// checks that Index counts 0, 1, 2, ... with no gap, that NodeAt inverts it, +// that a subtree's SubtreeEnd is one past its last index, and that namespace +// nodes have no index. +func checkIndex(t *testing.T, tr *Tree) { + t.Helper() + next := 0 + var walk func(n *Node) + walk = func(n *Node) { + if got := n.Index(); got != next { + t.Fatalf("%v %v: Index %d, want %d", n.Kind(), n.Name(), got, next) + } + if tr.NodeAt(next) != n { + t.Fatalf("NodeAt(%d) is not the node whose Index is %d", next, next) + } + next++ + for ns := range n.NamespaceNodes() { + if ns.Index() != -1 || ns.SubtreeEnd() != -1 { + t.Fatalf("namespace node has Index %d", ns.Index()) + } + } + for a := range n.Attrs() { + if a.Index() != next || a.SubtreeEnd() != next+1 { + t.Fatalf("attribute %v: Index %d End %d, want %d", a.Name(), a.Index(), a.SubtreeEnd(), next) + } + if tr.NodeAt(next) != a { + t.Fatalf("NodeAt(%d) is not the attribute", next) + } + next++ + } + for c := range n.Children() { + walk(c) + } + if n.SubtreeEnd() != next { + t.Fatalf("%v: SubtreeEnd %d, want %d", n.Name(), n.SubtreeEnd(), next) + } + } + walk(tr.Root) + if tr.Len() != next { + t.Fatalf("Len %d, want %d", tr.Len(), next) + } + if tr.NodeAt(-1) != nil || tr.NodeAt(next) != nil || tr.NodeAt(next+1<<20) != nil { + t.Fatal("NodeAt out of range is not nil") + } +} + +func TestNodeIndexDense(t *testing.T) { + var ns strings.Builder + ns.WriteString(``) + for i := 0; i < 50; i++ { + ns.WriteString(`t`) + } + ns.WriteString(``) + docs := map[string][]byte{"namespaces": []byte(ns.String())} + for name, glob := range map[string]string{ + "docbook": "xsltng/src/test/resources/xml/*.xml", + "xmark": "bench/xmark/auction-0.01.xml", + } { + paths, _ := filepath.Glob(filepath.Join("..", "testdata", filepath.FromSlash(glob))) + sort.Strings(paths) + if len(paths) > 20 { + paths = paths[:20] + } + for _, p := range paths { + if data, err := os.ReadFile(p); err == nil { + docs[name+":"+filepath.Base(p)] = data + } + } + } + for name, data := range docs { + tr, err := Parse(bytes.NewReader(data), ParseOptions{MaxBytes: -1}) + if err != nil { + continue // a corpus file that needs more than defaults + } + t.Run(name, func(t *testing.T) { checkIndex(t, tr) }) + } +} + +// A tree still being built numbers what it has so far, and taking back the +// last subtree frees its indices. +func TestNodeIndexBuilt(t *testing.T) { + root := NewNode(KindElement, QName{Local: "r"}, "") + a := root.AppendElement(QName{Local: "a"}) + if root.Index() != 0 || a.Index() != 1 || root.SubtreeEnd() != 2 { + t.Fatalf("index %d %d end %d", root.Index(), a.Index(), root.SubtreeEnd()) + } + tr := root.tree + if tr.NodeAt(1) != a || tr.NodeAt(2) != nil { + t.Fatal("NodeAt on a built tree") + } + b := root.AppendElement(QName{Local: "b"}) + if b.Index() != 2 || root.SubtreeEnd() != 3 || tr.Len() != 3 { + t.Fatalf("after append: %d end %d len %d", b.Index(), root.SubtreeEnd(), tr.Len()) + } + root.RemoveLastChild() + if tr.Len() != 2 || tr.NodeAt(2) != nil { + t.Fatalf("after take-back: len %d", tr.Len()) + } +} diff --git a/xdm/record.go b/xdm/record.go index b8844c83..0f75685e 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1469,6 +1469,53 @@ func (n *Node) Identity() IdentityKey { return IdentityKey{n} } // relative position must use Compare: this value says nothing across trees. func (n *Node) Order() int { return int(n.orderKey()) } +// Index returns n's position in its tree, or -1 for a namespace node. +// +// The document, element, attribute, text, comment and processing-instruction +// nodes of one tree are numbered 0, 1, 2, ... without gaps, in document order: +// an element comes before its attributes, which come before its children, and +// a parent before everything below it. A tree made by a constructor holds +// several parentless roots; they are numbered in the order they were made. +// Attributes have indices. Namespace nodes are made on demand, outside the +// numbering, and have none. Indices of different trees are unrelated. +// +// An index never changes once the node exists, so it is stable for the whole +// life of a parsed tree. A tree still being built only appends, so the indices +// it has handed out stay valid, except that taking back the last subtree +// appended (RemoveLastChild, ReplaceLastChild) frees its indices, and later nodes reuse them. Len +// grows as nodes are appended. +func (n *Node) Index() int { + if n.flags&fSide != 0 { + return -1 + } + return int(n.self) +} + +// SubtreeEnd returns one more than the largest Index in n's subtree, its +// attributes included, so that [n.Index(), n.SubtreeEnd()) is the numbering of +// the subtree and a table over it needs SubtreeEnd()-Index() entries. A leaf +// or attribute gives Index()+1. For a node still being built it is the end so +// far and grows as children are appended. A namespace node gives -1. +func (n *Node) SubtreeEnd() int { + if n.flags&fSide != 0 { + return -1 + } + return int(n.tree.endOf(n)) +} + +// Len returns the number of indexed nodes in the tree: valid Index values are +// 0 to Len()-1. For a tree still being built it is the count so far. +func (t *Tree) Len() int { return int(t.n) } + +// NodeAt returns the node whose Index is i, or nil when i is negative or not +// less than Len. It never returns a namespace node. +func (t *Tree) NodeAt(i int) *Node { + if i < 0 || i >= int(t.n) { + return nil + } + return t.rec(uint32(i)) +} + // generateID returns the string fn:generate-id() gives n: "N", the tree id, // "x", and n's position in the tree. // From b331aa11ccd8c6974ebde3ae1264e603fad00961 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:01:56 -0500 Subject: [PATCH 90/98] xslt: hold accumulator values in slices indexed by node position --- xslt/accumalloc_test.go | 60 +++++++++++++++++++++++++++++++++++++++++ xslt/accumulator.go | 46 +++++++++++++++++++++++-------- 2 files changed, 95 insertions(+), 11 deletions(-) create mode 100644 xslt/accumalloc_test.go diff --git a/xslt/accumalloc_test.go b/xslt/accumalloc_test.go new file mode 100644 index 00000000..c18cb5e4 --- /dev/null +++ b/xslt/accumalloc_test.go @@ -0,0 +1,60 @@ +package xslt + +import ( + "context" + gort "runtime" + "strconv" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// accumBytesPerNode is the bytes the whole transform allocates per source +// node for a scalar accumulator over n elements. +func accumBytesPerNode(t *testing.T, n int) float64 { + const sheet = ` + + + + + + + ` + st, err := xdm.ParseString(sheet, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + c, err := Compile(st.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + src, err := xdm.ParseString(""+strings.Repeat("t", n)+"", xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var m0, m1 gort.MemStats + gort.GC() + gort.ReadMemStats(&m0) + res, err := c.Transform(context.Background(), src.Root, TransformOptions{}) + if err != nil { + t.Fatal(err) + } + gort.ReadMemStats(&m1) + var b strings.Builder + if err := res.Serialize(&b); err != nil || b.String() != strconv.Itoa(n) { + t.Fatalf("got %q, %v", b.String(), err) + } + return float64(m1.TotalAlloc-m0.TotalAlloc) / float64(src.Len()) +} + +// The accumulator tables are slices sized once from the source tree, not maps: +// the whole transform costs about 445 bytes a node, against about 605 with +// maps. The bound sits between them. +func TestAccumulatorAllocPerNode(t *testing.T) { + if got := accumBytesPerNode(t, 50000); got > 520 { + t.Fatalf("%.0f bytes allocated per source node, want under 520", got) + } +} diff --git a/xslt/accumulator.go b/xslt/accumulator.go index efdd83ec..844850d0 100644 --- a/xslt/accumulator.go +++ b/xslt/accumulator.go @@ -68,10 +68,20 @@ type accumulatorRule struct { // that reads an accumulator inside a template rule linear rather than // quadratic. type accumulatorValues struct { - before map[*xdm.Node]xdm.Sequence - after map[*xdm.Node]xdm.Sequence + // The tables run over root's subtree, indexed by Index()-base, and are + // sized once from SubtreeEnd. state says which values are set: a value + // may be the empty sequence, and an after-value must not be readable + // before its subtree has finished. Attributes are never set. + base int + before, after []xdm.Sequence + state []uint8 } +const ( + accumBeforeSet = 1 << iota + accumAfterSet +) + // accumCacheKey identifies one accumulator over one tree. type accumCacheKey struct { name string @@ -350,9 +360,12 @@ func (rt *runtime) accumulatorValuesFor(def *accumulatorDef, root *xdm.Node, if v := rt.accumBuilding[key]; v != nil { return v, nil } + size := root.SubtreeEnd() - root.Index() vals := &accumulatorValues{ - before: map[*xdm.Node]xdm.Sequence{}, - after: map[*xdm.Node]xdm.Sequence{}, + base: root.Index(), + before: make([]xdm.Sequence, size), + after: make([]xdm.Sequence, size), + state: make([]uint8, size), } rt.accumBuilding[key] = vals defer delete(rt.accumBuilding, key) @@ -408,7 +421,15 @@ func (rt *runtime) accumulatorValuesFor(def *accumulatorDef, root *xdm.Node, if cur, err = extend(cur, next); err != nil { return err } - vals.before[n] = cur + i := n.Index() - vals.base + if i >= len(vals.state) { + // Only a tree still being built outgrows its size. + vals.before = append(vals.before, make([]xdm.Sequence, i+1-len(vals.before))...) + vals.after = append(vals.after, make([]xdm.Sequence, i+1-len(vals.after))...) + vals.state = append(vals.state, make([]uint8, i+1-len(vals.state))...) + } + vals.before[i] = cur + vals.state[i] |= accumBeforeSet // Attributes and namespaces are NOT visited. §18.2.4's formal model // defines an accumulator over the sequence of events a streamed // traversal of the tree produces, and those events are the start and @@ -438,7 +459,8 @@ func (rt *runtime) accumulatorValuesFor(def *accumulatorDef, root *xdm.Node, if cur, err = extend(cur, end); err != nil { return err } - vals.after[n] = cur + vals.after[i] = cur + vals.state[i] |= accumAfterSet return nil } if err := walk(root); err != nil { @@ -641,12 +663,14 @@ func fnAccumulator(rt *runtime, ctx *xpath.Context, args []xdm.Sequence, if err != nil { return nil, err } - table := vals.before + table, set := vals.before, uint8(accumBeforeSet) if post { - table = vals.after + table, set = vals.after, accumAfterSet } - v, ok := table[node] - if !ok { + // A namespace node has no index, and a node appended to the tree after + // the walk lies past the table; neither was visited. + i := node.Index() - vals.base + if node.Index() < 0 || i < 0 || i >= len(table) || vals.state[i]&set == 0 { // The walk visits every node the checks above let through, so a // missing entry means the walk is still in progress and has not // reached this value: the rule computing it is reading it. That @@ -657,7 +681,7 @@ func fnAccumulator(rt *runtime, ctx *xpath.Context, args []xdm.Sequence, "XTDE3400: accumulator %s is defined circularly: a rule reads "+ "the value of %s(%q) it is computing", def.name.Lexical(), fname, lex) } - return v, nil + return table[i], nil } // noteCopiedAccumulators records that copy was made from orig, so that an From 73249e20808d56f0f5e742451f19e95371735fe0 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:02:39 -0500 Subject: [PATCH 91/98] xdm: make namespace nodes without rebuilding the scope per node; add InScopeNamespaceBindings --- xdm/nsbindings.go | 52 +++++++++++++++++++++++++++ xdm/nsbindings_test.go | 80 ++++++++++++++++++++++++++++++++++++++++++ xdm/record.go | 23 ++++++------ 3 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 xdm/nsbindings.go create mode 100644 xdm/nsbindings_test.go diff --git a/xdm/nsbindings.go b/xdm/nsbindings.go new file mode 100644 index 00000000..db4df358 --- /dev/null +++ b/xdm/nsbindings.go @@ -0,0 +1,52 @@ +package xdm + +// InScopeNamespaceBindings iterates the prefix-to-URI bindings in scope on +// an element, the same set InScopeNamespaces returns, without making a +// namespace node for any of them or a map to hold them. Other kinds have +// none. +// +// Each in-scope prefix is yielded once, with the URI the innermost +// declaration gives it, and the xml prefix is always among them. A prefix +// undeclared by xmlns:p="" (or a default undeclared by xmlns="") is not +// yielded. The order is unspecified; use NamespaceNodes for the namespace +// axis' prefix order. +// +// The cost of each binding is a scan of the ones yielded before it, so the +// iteration suits a scope of the usual size. +func (n *Node) InScopeNamespaceBindings() func(yield func(prefix, uri string) bool) { + return func(yield func(prefix, uri string) bool) { + if n.kind != uint8(KindElement) || n.flags&fSide != 0 { + return + } + if !yield("xml", NSXML) { + return + } + var buf [16]string + seen := buf[:0] // prefixes already decided: yielded or undeclared + for cur := n; cur != nil; cur = cur.Parent() { + if cur.kind != uint8(KindElement) || cur.flags&fSide != 0 { + continue + } + f := cur.frame() + for i := len(f) - 1; i >= 0; i-- { // the last declaration wins + b := f[i] + if b.prefix == "xml" || containsString(seen, b.prefix) { + continue + } + seen = append(seen, b.prefix) + if b.uri != "" && !yield(b.prefix, b.uri) { + return + } + } + } + } +} + +func containsString(s []string, x string) bool { + for _, v := range s { + if v == x { + return true + } + } + return false +} diff --git a/xdm/nsbindings_test.go b/xdm/nsbindings_test.go new file mode 100644 index 00000000..ee0182e4 --- /dev/null +++ b/xdm/nsbindings_test.go @@ -0,0 +1,80 @@ +package xdm + +import ( + "maps" + "runtime" + "testing" +) + +func TestInScopeNamespaceBindings(t *testing.T) { + tr, err := ParseString(``, ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var walk func(n *Node) + walk = func(n *Node) { + if n.Kind() == KindElement { + got := map[string]string{} + for p, u := range n.InScopeNamespaceBindings() { + if _, dup := got[p]; dup { + t.Errorf("%s: prefix %q yielded twice", n.Name().Local, p) + } + got[p] = u + } + if want := n.InScopeNamespaces(); !maps.Equal(got, want) { + t.Errorf("%s: got %v, want %v", n.Name().Local, got, want) + } + } + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + walk(c) + } + } + walk(tr.Root) + // Yielding stops when asked, and no namespace node is made on the way. + r := tr.Root.FirstChild() + k := 0 + for range r.InScopeNamespaceBindings() { + k++ + break + } + if k != 1 { + t.Errorf("break did not stop the iteration") + } + if len(tr.side) != 0 { + t.Errorf("iterating made %d namespace nodes", len(tr.side)) + } + if a := testing.AllocsPerRun(10, func() { + for range r.InScopeNamespaceBindings() { + } + }); a != 0 { + t.Errorf("iteration allocated %v times", a) + } +} + +// Making the namespace nodes of an element must not rebuild its scope for +// each one: that was quadratic in the bindings in scope. +func TestNamespaceNodesAllocation(t *testing.T) { + doc := `` + tr, err := ParseString(doc, ParseOptions{}) + if err != nil { + t.Fatal(err) + } + r := tr.Root.FirstChild() + n := 0 + var a, b runtime.MemStats + runtime.ReadMemStats(&a) + for range r.NamespaceNodes() { + n++ + } + runtime.ReadMemStats(&b) + if n != 41 { + t.Fatalf("got %d namespace nodes, want 41", n) + } + if m := b.Mallocs - a.Mallocs; m > 200 { // a scope rebuild per node cost over 1000 + t.Errorf("%d allocations for 41 namespace nodes", m) + } +} diff --git a/xdm/record.go b/xdm/record.go index b8844c83..d9428596 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -779,7 +779,7 @@ func (n *Node) NumNamespaceDecls() int { return len(n.frame()) } // NamespaceDeclAt returns the namespace node for the i'th declaration held on // n itself. It is the node the namespace axis gives for that prefix. func (n *Node) NamespaceDeclAt(i int) *Node { - return n.sideNode(n.v0, uint32(i), n.self) + return n.sideNode(n.v0, uint32(i), n.self, -1) } // NamespaceDecls iterates over the namespace nodes for the declarations held @@ -1332,8 +1332,8 @@ func (n *Node) NamespaceNodes() iter.Seq[*Node] { if n.kind != uint8(KindElement) || n.flags&fSide != 0 { return } - for _, s := range n.scopeSlots() { - if !yield(n.sideNode(s.frame, s.index, s.owner)) { + for i, s := range n.scopeSlots() { + if !yield(n.sideNode(s.frame, s.index, s.owner, i)) { return } } @@ -1375,8 +1375,9 @@ func (n *Node) scopeSlots() []scopeSlot { // sideNode returns the namespace node of n for the binding at (frame, // index), making it on first use. Making it is safe from any number of -// goroutines reading one tree. -func (n *Node) sideNode(frame, index, owner uint32) *Node { +// goroutines reading one tree. slot is the node's place among n's, or -1 to +// work it out from n's scope, which costs a rebuild of that scope. +func (n *Node) sideNode(frame, index, owner uint32, slot int) *Node { t := n.tree var prefix string if index == xmlBinding { @@ -1393,14 +1394,16 @@ func (n *Node) sideNode(frame, index, owner uint32) *Node { // The slot places the node among n's others, by prefix, for document // order; an undeclaring declaration, which has no place on the axis, // takes the place its prefix would have. - slot := uint32(0) - for _, s := range n.scopeSlots() { - if s.prefix < prefix { - slot++ + if slot < 0 { + slot = 0 + for _, s := range n.scopeSlots() { + if s.prefix < prefix { + slot++ + } } } s := &Node{tree: t, kind: uint8(KindNamespace), flags: fSide, - self: uint32(len(t.side)), parent: n.self, end: slot, prev: owner, + self: uint32(len(t.side)), parent: n.self, end: uint32(slot), prev: owner, v0: frame, v1: index} if t.side == nil { t.side = map[sideKey]*Node{} From 28d41c8787091987abca638543e7b865ad28e5ac Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:03:25 -0500 Subject: [PATCH 92/98] xmltok: size a long text run in one step, so a 10 MiB value allocates 1.08x not 1.87x --- internal/xmltok/xmltok.go | 13 ++++++--- xdm/textgrow_test.go | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 xdm/textgrow_test.go diff --git a/internal/xmltok/xmltok.go b/internal/xmltok/xmltok.go index fe5ab2fe..5e996898 100644 --- a/internal/xmltok/xmltok.go +++ b/internal/xmltok/xmltok.go @@ -45,7 +45,6 @@ import ( "encoding/binary" "fmt" "io" - "slices" "strconv" "strings" "unicode/utf8" @@ -1038,9 +1037,17 @@ func (d *Decoder) text(quote byte, cdata bool) ([]byte, bool) { // A run past a window long that outgrows the scratch is // sized from what is left of the input, not by append's // 1.25x steps. ponytail: capped at 8x the scratch, so that a - // run early in a huge document takes at most that. + // run early in a huge document takes at most that; past half + // a MiB the cap is 128x, so that one long value is sized in + // one step rather than through a chain of them (1.9x its size + // at 10 MiB). A document of several such runs pays one + // scratch of up to what is left, once. if rest := d.Size - int(d.InputOffset()); n > cap(out)-len(out) && len(out) >= bufSize && rest > n { - out = slices.Grow(out, min(rest, 8*cap(out))) + step := 8 + if cap(out) >= 1<<19 { + step = 128 + } + out = append(make([]byte, 0, len(out)+min(rest, step*cap(out))), out...) } out = append(out, run[:n]...) if n == 1 { diff --git a/xdm/textgrow_test.go b/xdm/textgrow_test.go new file mode 100644 index 00000000..9b4432fc --- /dev/null +++ b/xdm/textgrow_test.go @@ -0,0 +1,55 @@ +package xdm + +import ( + "bytes" + "runtime" + "strings" + "sync" + "testing" +) + +// A document that is one long text run is parsed in about its own size of +// allocation: the decoder sizes the run's buffer from the input left rather +// than growing it through several steps of its own, and the tree keeps that +// buffer as the value's block. The bound is 1.25x; the growth steps alone +// were 1.7x at 10 MiB. +func TestParseLongTextAllocatesOnce(t *testing.T) { + const n = 10 << 20 + doc := []byte("" + strings.Repeat("QUJD", n/4) + "") + var a, b runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&a) + tr, err := Parse(bytes.NewReader(doc), ParseOptions{}) + runtime.ReadMemStats(&b) + if err != nil { + t.Fatal(err) + } + if got := b.TotalAlloc - a.TotalAlloc; got > n+n/4 { + t.Errorf("parse allocated %d bytes for %d of text (%.2fx), want at most 1.25x", got, n, float64(got)/n) + } + if v := tr.Root.StringValue(); len(v) != n { + t.Errorf("text is %d bytes, want %d", len(v), n) + } +} + +// Concurrent parses of long text share nothing: each decoder's buffer is its +// own, and a value handed to a tree is not written again. +func TestParseLongTextConcurrent(t *testing.T) { + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + text := strings.Repeat("QUJD", (1<<20+i*997)/4) + tr, err := Parse(strings.NewReader(""+text+""), ParseOptions{}) + if err != nil { + t.Error(err) + return + } + if tr.Root.StringValue() != text { + t.Errorf("parse %d: text differs", i) + } + }() + } + wg.Wait() +} From 127b2f2ce7098e8cd75ab4c861d1f86da475961f Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:04:45 -0500 Subject: [PATCH 93/98] c14n: pool the output buffer instead of allocating 80 KiB per call --- c14n/canon.go | 31 ++++++++++++++++++++++++++++--- c14n/digest_alloc_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 c14n/digest_alloc_test.go diff --git a/c14n/canon.go b/c14n/canon.go index fe6d4425..e93fc5ca 100644 --- a/c14n/canon.go +++ b/c14n/canon.go @@ -6,6 +6,7 @@ import ( "regexp" "slices" "strings" + "sync" "github.com/knroy/go-xml/v2/xdm" ) @@ -96,7 +97,10 @@ type canon struct { const noNode = "\x00" func canonicalize(w io.Writer, ns NodeSet, opts Options) error { - _, err := run(w, ns, opts) + c, err := run(w, ns, opts) + if c != nil { + c.w.release() + } return err } @@ -108,7 +112,6 @@ func run(w io.Writer, ns NodeSet, opts Options) (*canon, error) { } _, all := ns.(subtree) c := &canon{ - w: newOutBuf(w), set: ns, all: all, excl: opts.Algorithm.Exclusive(), @@ -139,6 +142,7 @@ func run(w io.Writer, ns NodeSet, opts Options) (*canon, error) { c.scope.m[prefix] = uri } } + c.w = newOutBuf(w) var err error switch root.Kind() { case xdm.KindDocument: @@ -150,6 +154,7 @@ func run(w io.Writer, ns NodeSet, opts Options) (*canon, error) { c.leaf(root) } default: + c.w.release() return nil, ErrUnsupportedNode } if err == nil { @@ -170,12 +175,32 @@ func run(w io.Writer, ns NodeSet, opts Options) (*canon, error) { // into nothing. type outBuf struct { buf []byte + pb *[]byte // the pool entry buf came from w io.Writer err error } +// outBufPool keeps the buffers between calls: a fixed 80 KiB allocated per +// call was 81 KB of garbage for each Digest of a tiny document. +var outBufPool = sync.Pool{New: func() any { + b := make([]byte, 0, outBufSize+outBufSize/4) + return &b +}} + func newOutBuf(w io.Writer) *outBuf { - return &outBuf{buf: make([]byte, 0, outBufSize+outBufSize/4), w: w} + pb := outBufPool.Get().(*[]byte) + return &outBuf{buf: (*pb)[:0], pb: pb, w: w} +} + +// release returns the buffer to the pool unless output grew it far past its +// size, so that one large text node does not stay allocated. o is not used +// afterwards. +func (o *outBuf) release() { + if cap(o.buf) <= 2*outBufSize { + *o.pb = o.buf[:0] + outBufPool.Put(o.pb) + } + o.buf, o.pb = nil, nil } // flush hands buf to w. A short write without an error is io.ErrShortWrite, diff --git a/c14n/digest_alloc_test.go b/c14n/digest_alloc_test.go new file mode 100644 index 00000000..f2541039 --- /dev/null +++ b/c14n/digest_alloc_test.go @@ -0,0 +1,27 @@ +package c14n + +import ( + "crypto/sha256" + "testing" +) + +// Every call allocated an 80 KiB output buffer, 81 KB of garbage for each +// Digest of a tiny document. The buffers now come from a pool. The bound is half the unpooled +// figure because the race detector makes a pool drop a quarter of its puts. +func TestDigestSmallDocumentAllocation(t *testing.T) { + root := parse(t, `x`).FirstChild() + h := sha256.New() + const calls = 101 + got := heapAllocated(func() { + for range calls { + h.Reset() + if _, err := Digest(h, root, Options{Algorithm: Inclusive10}); err != nil { + t.Fatal(err) + } + } + }) + t.Logf("%d Digest calls allocated %d bytes", calls, got) + if got > 4<<20 { + t.Errorf("%d Digest calls allocated %d bytes, want under 4 MiB (8.3 MB with a fresh buffer each)", calls, got) + } +} From 5d6454ad1dbf77bb93ccc75f2d3de01e724a472e Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:04:45 -0500 Subject: [PATCH 94/98] c14n: size Bytes and BytesNodeSet output from the tree for Subtree and ExcludeSubtree --- c14n/bytes_alloc_test.go | 10 ++--- c14n/c14n.go | 86 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/c14n/bytes_alloc_test.go b/c14n/bytes_alloc_test.go index a43bf5e0..55167720 100644 --- a/c14n/bytes_alloc_test.go +++ b/c14n/bytes_alloc_test.go @@ -6,9 +6,9 @@ import ( ) // Bytes and BytesNodeSet grew a bytes.Buffer by doubling from empty, 3.7 -// times the canonical form in all, and Equal paid it for its first operand. -// They now keep the writer's chunks and join them once, about twice the -// output. +// times the canonical form in all, then kept the writer's chunks and joined +// them, 2.1 times, and Equal paid it for its first operand. They now size +// the result from the node set and write into it once, about 1.1 times. func TestBytesAllocation(t *testing.T) { // 3 MB: doubling cost 3.1x the output here and 3.7x at 10 MB. root := cachedDoc(t, 3<<20).FirstChild() @@ -26,8 +26,8 @@ func TestBytesAllocation(t *testing.T) { } { got := heapAllocated(tc.f) t.Logf("%s: %d bytes allocated for %d bytes of output (%.2fx)", tc.name, got, len(out), float64(got)/float64(len(out))) - if got > uint64(len(out))*5/2 { - t.Errorf("%s allocated %d bytes for %d bytes of output, want under 2.5x", tc.name, got, len(out)) + if got > uint64(len(out))*7/5 { + t.Errorf("%s allocated %d bytes for %d bytes of output, want under 1.4x", tc.name, got, len(out)) } } } diff --git a/c14n/c14n.go b/c14n/c14n.go index e0d765f2..4eda2bd0 100644 --- a/c14n/c14n.go +++ b/c14n/c14n.go @@ -192,12 +192,10 @@ func FormatPrefixList(prefixes []string) string { // descendants are, with the namespace and xml:* axis treatment the // chosen algorithm specifies for a subset apex. func Bytes(n *xdm.Node, opts Options) ([]byte, error) { - // The output arrives in chunks of about outBufSize. They are kept and - // joined once at the end, about twice the canonical form in all, where a - // bytes.Buffer grown by doubling from empty cost up to 3.6 times. - var c chunkWriter - err := Write(&c, n, opts) - return c.bytes(), err + if n == nil || (n.Kind() != xdm.KindDocument && n.Kind() != xdm.KindElement) { + return nil, ErrUnsupportedNode + } + return BytesNodeSet(Subtree(n), opts) } // chunkWriter keeps a copy of everything written to it. @@ -219,6 +217,72 @@ func (c *chunkWriter) bytes() []byte { return bytes.Join(c.chunks, nil) } +// sizedWriter collects everything written to it in one buffer, sized once +// from the tree under the node set's root at the first write (after the options were validated, so +// a refused call allocates nothing). The estimate runs 5% to 15% over the +// output, so the result costs about 1.1 times its size where a buffer grown +// by doubling cost 2.1 times and joined chunks twice; a short estimate (an +// escaped value is longer than its text) costs one more copy. +type sizedWriter struct { + ns NodeSet + opts Options + buf []byte +} + +func (s *sizedWriter) Write(p []byte) (int, error) { + if s.buf == nil { + s.buf = make([]byte, 0, max(estimateSize(s.ns, s.opts), len(p))) + } + if need := len(s.buf) + len(p); need > cap(s.buf) { + // A short estimate: grow by a quarter, not by append's doubling. + s.buf = append(make([]byte, 0, need+need/4), s.buf...) + } + s.buf = append(s.buf, p...) + return len(p), nil +} + +// estimateSize adds up the markup and values of ns's members. It reads the +// tree once, a sixth of the time canonicalizing it takes. It counts every +// declaration an element has and no escaping, and adds an eighth for the +// escapes, so it is an estimate. It does not ask ns which nodes are members +// (a NodeSet is asked once per node), so it is for the sets this package +// made: a Subtree is counted exactly, an ExcludeSubtree with what it excludes. +func estimateSize(ns NodeSet, opts Options) int { + root := ns.Root() + if root == nil { + return 0 + } + comments := opts.Algorithm.WithComments() + n := 0 + visit := func(d *xdm.Node) { + switch d.Kind() { + case xdm.KindElement: + q := d.Name() + n += 2*(len(q.Local)+len(q.Prefix)) + 6 + for a := range d.Attrs() { + q := a.Name() + n += len(q.Local) + len(q.Prefix) + len(a.Value()) + 5 + } + for p, u := range d.DeclaredNamespaces() { + n += len(p) + len(u) + 9 + } + case xdm.KindComment: + if comments { + n += len(d.Value()) + 8 + } + case xdm.KindPI: + n += len(d.Name().Local) + len(d.Value()) + 8 + case xdm.KindText: + n += len(d.Value()) + } + } + visit(root) + for d := range root.Descendants() { + visit(d) + } + return n + n/8 +} + // Digest canonicalizes n and writes the result into h, returning h's sum. // // This is the form signature verification wants. It streams, so the @@ -299,6 +363,16 @@ func WriteNodeSet(w io.Writer, ns NodeSet, opts Options) error { // BytesNodeSet mirrors Bytes for a subset. func BytesNodeSet(ns NodeSet, opts Options) ([]byte, error) { + switch ns.(type) { + case subtree, excludeSet: + c := sizedWriter{ns: ns, opts: opts} + err := WriteNodeSet(&c, ns, opts) + return c.buf, err + } + // Any other set is the caller's, and Contains is called once per node: + // the output cannot be sized without calling it again, so it arrives in + // chunks of about outBufSize, kept and joined once, about twice the + // canonical form in all. var c chunkWriter err := WriteNodeSet(&c, ns, opts) return c.bytes(), err From 22d32acb4f74098a2974811beb18cb1b9f1ffac8 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:07:41 -0500 Subject: [PATCH 95/98] xdm, xmltok: size the per-attribute structures of a many-attribute tag in one step (99x to 38x at 10 MiB) --- internal/xmltok/xmltok.go | 23 +++++++++++++++++++++++ xdm/parse.go | 3 +++ xdm/record.go | 31 +++++++++++++++++++++++++++++++ xdm/textgrow_test.go | 32 ++++++++++++++++++++++++++++++++ xdm/wellformed.go | 2 +- 5 files changed, 90 insertions(+), 1 deletion(-) diff --git a/internal/xmltok/xmltok.go b/internal/xmltok/xmltok.go index 5e996898..987dd68b 100644 --- a/internal/xmltok/xmltok.go +++ b/internal/xmltok/xmltok.go @@ -423,6 +423,7 @@ func (d *Decoder) startTag() (Token, error) { return nil, d.orSyntaxError("expected element name after <") } attrs := d.attrs[:0] + start := d.InputOffset() for { spaced := d.space() b, ok := d.mustgetc() @@ -474,6 +475,9 @@ func (d *Decoder) startTag() (Token, error) { // to, go on to full blocks. d.arena.buf = make([]byte, 0, min(d.Size, arenaBlock)) } + if len(attrs) == cap(attrs) && len(attrs) >= bigTag { + attrs = d.growAttrs(attrs, start) + } attrs = append(attrs, Attr{an, d.arena.String(v)}) } d.attrs = attrs @@ -481,6 +485,25 @@ func (d *Decoder) startTag() (Token, error) { return &d.tokStart, nil } +// bigTag is how many attributes a start tag has before startTag stops growing +// the slice of them by append's steps (1.25x at this size, so a chain of them +// allocates 5x the final slice). +const bigTag = 256 + +// growAttrs makes room in attrs for the rest of a tag with many attributes. +// The input left, over the bytes an attribute has taken so far, bounds how many +// remain; ponytail: capped at 8x the slice, so that a tag early in a huge +// document takes at most that, and at least doubling. +func (d *Decoder) growAttrs(attrs []Attr, start int64) []Attr { + est := len(attrs) + if rest := d.Size - int(d.InputOffset()); rest > 0 { + if used := int(d.InputOffset() - start); used > 0 { + est = rest / max(used/len(attrs), 1) + } + } + return append(make([]Attr, 0, len(attrs)+max(len(attrs), min(est, 8*len(attrs)))), attrs...) +} + // procInst reads [16] PI after "", and the body is checked // against [2] Char. diff --git a/xdm/parse.go b/xdm/parse.go index 54edbb9e..6fbcb9ba 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -835,6 +835,9 @@ func buildElement(t xml.StartElement, parent *Node, ns *nsScope, names *nameCach // validateStartElement has refused an unbound prefix. uri, _ := ns.lookup(t.Name.Space) el.name = names.intern(tree, QName{Prefix: t.Name.Space, Local: t.Name.Local, URI: uri}) + if len(t.Attr) >= bigTag { + tree.reserveNames(len(t.Attr)) + } for _, a := range t.Attr { if a.Name.Space == "xmlns" || (a.Name.Space == "" && a.Name.Local == "xmlns") { continue diff --git a/xdm/record.go b/xdm/record.go index b8844c83..eb7a7bc7 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -3,8 +3,10 @@ package xdm import ( "bytes" "iter" + "maps" "math" "math/bits" + "slices" "sort" "strconv" "sync" @@ -406,6 +408,35 @@ func (t *Tree) intern(q QName) uint32 { return i } +// bigTag is how many attributes a start tag carries before buildElement sizes +// the name table for them in one step, rather than letting each new name +// grow it. +const bigTag = 256 + +// reserveNames makes room for n more names, the most a start tag of n +// attributes can add, so that a tag of many distinct names grows the table +// and its index once instead of through every doubling. The index is +// rebuilt only when n would at least double it, so a run of big tags that +// repeat their names rebuilds nothing. +func (t *Tree) reserveNames(n int) { + if t.names == nil { + t.names = make([]QName, 1, n+1) + } else { + t.names = slices.Grow(t.names, n) + } + if t.nameIx == nil { + t.nameIx = make(map[QName]uint32, len(t.names)+n) + t.nameHot = new([64]uint32) + for j, x := range t.names[1:] { + t.nameIx[x] = uint32(j + 1) + } + } else if n > len(t.nameIx) { + ix := make(map[QName]uint32, len(t.nameIx)+n) + maps.Copy(ix, t.nameIx) + t.nameIx = ix + } +} + // indexNames builds the name index over the names the tree holds. func (t *Tree) indexNames() { t.nameIx = make(map[QName]uint32, 2*len(t.names)) diff --git a/xdm/textgrow_test.go b/xdm/textgrow_test.go index 9b4432fc..af0e2aaa 100644 --- a/xdm/textgrow_test.go +++ b/xdm/textgrow_test.go @@ -3,6 +3,7 @@ package xdm import ( "bytes" "runtime" + "strconv" "strings" "sync" "testing" @@ -53,3 +54,34 @@ func TestParseLongTextConcurrent(t *testing.T) { } wg.Wait() } + +// A start tag of many distinct attributes sizes the structures that grow with +// them -- the tokenizer's attribute slice, the name table and its index, the +// duplicate check -- from the tag, not through every step of their growth. +// 100,000 attributes allocated about 100 bytes of each byte of the tag with +// the steps; the structures' own floor is about 40. +func TestParseManyAttributesAllocation(t *testing.T) { + var sb strings.Builder + sb.WriteString("") + doc := sb.String() + var a, b runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&a) + tr, err := Parse(strings.NewReader(doc), ParseOptions{}) + runtime.ReadMemStats(&b) + if err != nil { + t.Fatal(err) + } + if got := tr.Root.ChildElements()[0].NumAttrs(); got != 100000 { + t.Fatalf("%d attributes, want 100000", got) + } + if got := (b.TotalAlloc - a.TotalAlloc) / uint64(len(doc)); got > 60 { + t.Errorf("parse allocated %dx the tag's size, want at most 60x", got) + } +} diff --git a/xdm/wellformed.go b/xdm/wellformed.go index 6841e6f1..e5ec446a 100644 --- a/xdm/wellformed.go +++ b/xdm/wellformed.go @@ -221,7 +221,7 @@ func validateStartElement(t xml.StartElement, ns *nsScope, xml11 bool) error { case len(seen) < len(small): seen = append(seen, k) default: - many = make(map[expandedName]bool, 2*len(small)) + many = make(map[expandedName]bool, max(2*len(small), len(t.Attr))) for _, s := range seen { many[s] = true } From 567aa1a69c4bf02ade21926597afc3751c6092bf Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:09:06 -0500 Subject: [PATCH 96/98] xpath: fewer allocations per predicate step The comparison hoist now lives on the stack and an ancestor step collects its chain in one block of four instead of growing 1, 2, 4. --- xpath/eval.go | 29 +++++++++++++++--------- xpath/predicate_allocs_test.go | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 10 deletions(-) create mode 100644 xpath/predicate_allocs_test.go diff --git a/xpath/eval.go b/xpath/eval.go index 10f799ba..85c9d797 100644 --- a/xpath/eval.go +++ b/xpath/eval.go @@ -121,8 +121,14 @@ func (e *Step) appendFrom(selected xdm.Sequence, node *xdm.Node) xdm.Sequence { } return appendNamedDescendants(selected, node, nt) } + // Ancestors come in a chain, so a first match usually has more behind it: + // one block of four in place of the 1, 2, 4 growth. + chain := e.Axis == AxisAncestor || e.Axis == AxisAncestorOrSelf walkAxis(node, e.Axis, func(n *xdm.Node) bool { if e.Test.Matches(n, principal) { + if chain && selected == nil { + selected = make(xdm.Sequence, 0, 4) + } selected = append(selected, n) } return true @@ -360,8 +366,9 @@ func applyPredicate(ctx *Context, seq xdm.Sequence, pred Expr) (xdm.Sequence, er var out xdm.Sequence size := len(seq) var h *comparisonHoist - if size > 1 { - h = newComparisonHoist(ctx, pred) + var hoist comparisonHoist // on the stack: one allocation less per predicate + if size > 1 && hoist.init(ctx, pred) { + h = &hoist } // One focus context serves every item when the predicate cannot keep @@ -502,34 +509,36 @@ type comparisonHoist struct { ready bool } -// newComparisonHoist returns nil when pred is not such a comparison, under +// init sets up h for pred and reports true, or reports false when pred is not +// such a comparison, under // XPath 1.0 compatibility (whose conversions need the raw operands), and for // "A = (M to N)", which evalGeneralComparison answers from the bounds. -func newComparisonHoist(ctx *Context, pred Expr) *comparisonHoist { +func (h *comparisonHoist) init(ctx *Context, pred Expr) bool { b, ok := pred.(*BinaryOp) if !ok { - return nil + return false } switch b.Op { case "eq", "ne", "lt", "le", "gt", "ge": case "=", "!=", "<", "<=", ">", ">=": if r, isRange := b.Right.(*BinaryOp); ctx.Compat() || (isRange && r.Op == "to") { - return nil + return false } default: - return nil + return false } side, left := b.Right, false if !hoistable(side, false) { side, left = b.Left, true if !hoistable(side, false) { - return nil + return false } } if !callsBuiltins(side, ctx) { - return nil + return false } - return &comparisonHoist{cmp: b, left: left} + *h = comparisonHoist{cmp: b, left: left} + return true } // holds reports whether the predicate selects sub's context item. The hoisted diff --git a/xpath/predicate_allocs_test.go b/xpath/predicate_allocs_test.go new file mode 100644 index 00000000..cff95d22 --- /dev/null +++ b/xpath/predicate_allocs_test.go @@ -0,0 +1,40 @@ +package xpath + +import ( + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// TestPredicateStepAllocs pins the allocations of the XML-DSig transform shape, +// an ancestor step with a comparison predicate evaluated once per node on one +// reused context: the comparison hoist lives on the stack and the ancestor +// chain is collected in one block. Measured at 8 per evaluation here; 11 without them. +func TestPredicateStepAllocs(t *testing.T) { + tree, err := xdm.ParseString(``, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var leaf *xdm.Node + for n := tree.Root; n != nil; { + leaf = n + var next *xdm.Node + for c := range n.Children() { + next = c + } + n = next + } + c, err := Compile(`ancestor-or-self::*[@Id='x']`, nil) + if err != nil { + t.Fatal(err) + } + ctx := NewContext(leaf, Builtins()) + allocs := testing.AllocsPerRun(50, func() { + if _, err := c.Eval(ctx); err != nil { + t.Fatal(err) + } + }) + if allocs > 9 { + t.Errorf("%v allocations per evaluation, want at most 9", allocs) + } +} From 63e509a6d2bf64eec8a514eae96bc6ed939736b6 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:20:32 -0500 Subject: [PATCH 97/98] Revert "xmltok: size a long text run in one step, so a 10 MiB value allocates 1.08x not 1.87x" This reverts commit 28d41c8787091987abca638543e7b865ad28e5ac. --- internal/xmltok/xmltok.go | 13 +++---------- xdm/textgrow_test.go | 25 ------------------------- 2 files changed, 3 insertions(+), 35 deletions(-) diff --git a/internal/xmltok/xmltok.go b/internal/xmltok/xmltok.go index 987dd68b..3a85fd5e 100644 --- a/internal/xmltok/xmltok.go +++ b/internal/xmltok/xmltok.go @@ -45,6 +45,7 @@ import ( "encoding/binary" "fmt" "io" + "slices" "strconv" "strings" "unicode/utf8" @@ -1060,17 +1061,9 @@ func (d *Decoder) text(quote byte, cdata bool) ([]byte, bool) { // A run past a window long that outgrows the scratch is // sized from what is left of the input, not by append's // 1.25x steps. ponytail: capped at 8x the scratch, so that a - // run early in a huge document takes at most that; past half - // a MiB the cap is 128x, so that one long value is sized in - // one step rather than through a chain of them (1.9x its size - // at 10 MiB). A document of several such runs pays one - // scratch of up to what is left, once. + // run early in a huge document takes at most that. if rest := d.Size - int(d.InputOffset()); n > cap(out)-len(out) && len(out) >= bufSize && rest > n { - step := 8 - if cap(out) >= 1<<19 { - step = 128 - } - out = append(make([]byte, 0, len(out)+min(rest, step*cap(out))), out...) + out = slices.Grow(out, min(rest, 8*cap(out))) } out = append(out, run[:n]...) if n == 1 { diff --git a/xdm/textgrow_test.go b/xdm/textgrow_test.go index af0e2aaa..c2d1432b 100644 --- a/xdm/textgrow_test.go +++ b/xdm/textgrow_test.go @@ -1,7 +1,6 @@ package xdm import ( - "bytes" "runtime" "strconv" "strings" @@ -9,30 +8,6 @@ import ( "testing" ) -// A document that is one long text run is parsed in about its own size of -// allocation: the decoder sizes the run's buffer from the input left rather -// than growing it through several steps of its own, and the tree keeps that -// buffer as the value's block. The bound is 1.25x; the growth steps alone -// were 1.7x at 10 MiB. -func TestParseLongTextAllocatesOnce(t *testing.T) { - const n = 10 << 20 - doc := []byte("" + strings.Repeat("QUJD", n/4) + "") - var a, b runtime.MemStats - runtime.GC() - runtime.ReadMemStats(&a) - tr, err := Parse(bytes.NewReader(doc), ParseOptions{}) - runtime.ReadMemStats(&b) - if err != nil { - t.Fatal(err) - } - if got := b.TotalAlloc - a.TotalAlloc; got > n+n/4 { - t.Errorf("parse allocated %d bytes for %d of text (%.2fx), want at most 1.25x", got, n, float64(got)/n) - } - if v := tr.Root.StringValue(); len(v) != n { - t.Errorf("text is %d bytes, want %d", len(v), n) - } -} - // Concurrent parses of long text share nothing: each decoder's buffer is its // own, and a value handed to a tree is not written again. func TestParseLongTextConcurrent(t *testing.T) { From c58525b5c13f250a65cb64796e25b0d4faea4351 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 01:21:43 -0500 Subject: [PATCH 98/98] docs: record the go-xmlsec memory fixes Changelog rows for the node index and namespace binding APIs, the c14n buffer and sizing, many-attribute tags, namespace nodes, accumulator slices and predicate steps; the reverted long-text change and caller node sets in known-gaps; the WithVersion advice; regenerated test counts. --- CHANGELOG.md | 13 +++++++++++++ README.md | 4 ++-- docs/c14n.md | 2 +- docs/conformance-gaps.md | 2 +- docs/known-gaps.md | 10 ++++++++-- docs/options.md | 6 +++++- docs/stats.md | 2 +- docs/testing.md | 2 +- docs/todo.md | 2 +- 9 files changed, 33 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9fa0b54..b0d0f420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ Three new exported functions make this a minor release. |---|---|---| | `xpath.SerializeJSONTo`, `xpath.SerializeAdaptiveTo` | The json and adaptive methods written to an `io.Writer` instead of returned as a string. | [`599b2f9`][599b2f9] | | `xdm.Node.AddNamespaces` | Sets several namespace bindings on an element in one call. | [`b42f04a`][b42f04a] | +| `xdm.Node.Index`, `Node.SubtreeEnd`, `Tree.Len`, `Tree.NodeAt` | A node's dense, document-order position in its tree (attributes included, namespace nodes -1), stable once the node exists; size a table with `SubtreeEnd() - Index()` and look nodes up by position. | [`b37796b`][b37796b] | +| `xdm.Node.InScopeNamespaceBindings` | Iterates an element's in-scope prefixes and URIs without making a namespace node for each. | [`73249e2`][73249e2] | | `xdm.ArrayItem.AppendMembers` | Appends an array's members to a slice without an intermediate copy. | [`59ac569`][59ac569] | | `xpath.CompileBounded`, `xpath.MaxRegexpProgram` | `regexp.Compile` that refuses a program over 100,000 instructions; used by XPath and XSD patterns. | [`c1c2bb9`][c1c2bb9] | | `xpath.Env.MaxDigits`, `xpath.MaxDigits` | Bounds the digits of an `xs:integer` or `xs:decimal` cast or arithmetic result; 0 is 10^6, negative is none. | [`8577906`][8577906] | @@ -77,6 +79,10 @@ Three new exported functions make this a minor release. | Dynamic calls, FLWOR tuples and window clauses made a scope per variable; closures kept their whole scope | One scope per binding set, and a closure keeps only what its body reads: `$f($i)` over 1M 2,673 → 1,617 MB. | [`5951f24`][5951f24], [`a4f316d`][a4f316d], [`b5aaf06`][b5aaf06] | | The function result memo kept each key string | It keeps a 128-bit SHA-256 hash: DocBook `table-cals.049` live heap 359 → 172 MB, peak 1,150 → 533 MB. | [`15273c7`][15273c7] | | `fn:doc` read the file whole and copied it; `replace` copied and over-charged a non-matching input | `fn:doc` parses as it reads (16 MB file 82 → 29 MB); `replace` returns early with no match (64 MB: 256 → 0 MB). | [`a6f2eaa`][a6f2eaa], [`6bab64a`][6bab64a] | +| Every c14n call allocated an 80 KiB buffer, and `Bytes` grew its result by doubling | The buffer is pooled (101 `Digest` calls on a tiny document 8.4 MB → 121 KB); `Bytes` and package-made node sets write into a buffer sized from the tree (about 1.1× the output, from 2.1–3.6×). | [`127b2f2`][127b2f2], [`5d6454a`][5d6454a] | +| A tag with many unique attribute names grew its slices, name table and duplicate map step by step | They are sized in one step past 256 attributes: 10 MiB of unique attributes 99.5× → 37.6× allocated. Small documents are unchanged. | [`22d32ac`][22d32ac] | +| Each namespace node rebuilt its element's whole scope to find its place; accumulators kept two maps per node | The slot is passed in (c14n over an XPath node set at 1 MiB 356 → 299 MB); accumulator values sit in slices indexed by `Node.Index` (about 27% less per node). | [`73249e2`][73249e2], [`b331aa1`][b331aa1] | +| A predicate step heap-allocated its comparison, and an ancestor step grew its result one node at a time | Both are cut: `ancestor-or-self::*[@Id='x']` per node 384 B / 9.75 allocations → 304 B / 7. | [`567aa1a`][567aa1a] | | `xsl:sort`, `xsl:iterate`, `xsl:result-document`, built-in shallow-copy, namespace scopes and `on-empty` allocated per item | Smaller sort values in one array, parameter slices, set lookups, no focus copy without rules, frame walks, a lazy leaf walk. `result-document` is linear. | [`6ea432f`][6ea432f], [`a3d8ef6`][a3d8ef6], [`951f0d9`][951f0d9], [`f650a19`][f650a19], [`4b0b564`][4b0b564], [`469c95e`][469c95e] | | DTD defaults were scanned per element; XSD key scopes rebuilt their entries per level | Defaults indexed by element (188 → 5 MB); scopes extend in place (960 levels 84 → 1 MB). | [`6293e98`][6293e98], [`996b7db`][996b7db] | | Large regex programs were cached; `parse-json` objects and `serialize` refusals cost per key and item | Programs over 4,096 instructions are not cached; objects use `MapBuilder` (about 3.3× less); `serialize` stops at the first refusal. | [`331bc96`][331bc96], [`98811f6`][98811f6], [`6317d49`][6317d49] | @@ -1789,3 +1795,10 @@ here so every entry in this file sits under a release. [331bc96]: https://github.com/knroy/go-xml/commit/331bc96 [98811f6]: https://github.com/knroy/go-xml/commit/98811f6 [6317d49]: https://github.com/knroy/go-xml/commit/6317d49 +[b37796b]: https://github.com/knroy/go-xml/commit/b37796b +[73249e2]: https://github.com/knroy/go-xml/commit/73249e2 +[127b2f2]: https://github.com/knroy/go-xml/commit/127b2f2 +[5d6454a]: https://github.com/knroy/go-xml/commit/5d6454a +[22d32ac]: https://github.com/knroy/go-xml/commit/22d32ac +[b331aa1]: https://github.com/knroy/go-xml/commit/b331aa1 +[567aa1a]: https://github.com/knroy/go-xml/commit/567aa1a diff --git a/README.md b/README.md index a43a21ae..6e6d2da2 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ and maintains it as a project of his own. | **XSD 1.1** | 99.98% instance (26,217 of 26,222); **99.97%** schema-validity (15,350 of 15,354); opt-in via `Version11` | | **RELAX NG** | 100.00% of James Clark's spectest (965 of 965 assertions); XML and compact syntax, from Go and from `go-xml validate -rng` | | **DTD** | content models, attribute defaults, enumerations, `ID`/`IDREF`; external subset, parameter entities across both subsets, conditional sections — via `dtd.Load` with a caller-supplied resolver, nothing fetched by default | -| **Tests** | 2,941 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | +| **Tests** | 2,950 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | | **Production schemas** | UBL 2.1, UN/CEFACT CII, Factur-X/ZUGFeRD, Peppol BIS 3.0 — 88 schemas load, instances validate clean | | **API** | 1.2; the exported surface is stable and additive over 1.1, and a breaking change means 2.0 with a new module path | @@ -1388,7 +1388,7 @@ back, is in [docs/testing.md](docs/testing.md). | method | what it catches | what it misses | |---|---|---| -| **Unit tests** (2,941 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** (2,950 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | | **Spec inventories** | features absent entirely | features present but behaving wrongly | | **Saxon differential** | subtle behavioural divergence on real stylesheets | constructs the corpora do not use | | **W3C QT3 suite** | systematic conformance across 22,054 XPath and 30,517 XQuery cases | XSLT (it is an XPath suite) | diff --git a/docs/c14n.md b/docs/c14n.md index 0a27e58e..89ed7bc4 100644 --- a/docs/c14n.md +++ b/docs/c14n.md @@ -409,7 +409,7 @@ against `xdm`'s public API as it stands: | R1 | The namespace declarations made *on* each element, as written | `Node.NamespaceDecls()` — the declarations on that element only, an undeclaration (`xmlns=""`) carried as an empty value | yes | | R2 | The bindings *in scope* at an element | `Node.InScopeNamespaces()` (a prefix-to-URI map, `xml` always present) and `Node.LookupPrefix(prefix)` for one prefix | yes | | R3 | `xml:*` attributes, for inheritance and the 1.1 `xml:base` fix-up | ordinary `Node.Attrs()` whose `Name().URI == xdm.NSXML` | yes | -| R4 | Document order | `Node.Compare` orders any two nodes. `Node.Order()` is a per-tree index assigned by `Tree.Finalize` in one pre-order walk, which every parser entry point calls; it is strictly increasing in document order within a finalized tree but **not dense** — `Finalize` reserves a slot for every in-scope binding of each element, for the namespace-axis nodes synthesized on demand, and adds a per-tree offset. `c14n` needs neither: it walks `Children` and `Attrs` in order and sorts what the specifications tell it to sort | yes | +| R4 | Document order | `Node.Compare` orders any two nodes. `Node.Order()` is a per-tree index assigned by `Tree.Finalize` in one pre-order walk, which every parser entry point calls; it is strictly increasing in document order within a finalized tree but **not dense** — `Finalize` reserves a slot for every in-scope binding of each element, for the namespace-axis nodes synthesized on demand, and adds a per-tree offset. `Node.Index()` is the dense position, with `Tree.NodeAt` to go back. `c14n` needs neither: it walks `Children` and `Attrs` in order and sorts what the specifications tell it to sort | yes | | R5 | Node identity, for node-set membership | pointer equality for every node the parser builds; `Node.Is` for the general case (it differs only for synthesized namespace nodes, which never reach `c14n`) | yes | | R6 | Character data in canonical shape | the parser resolves character and entity references, merges CDATA sections with adjacent text into one text node, normalises CRLF and a lone CR to LF, keeps a ` ` reference as a literal CR (which C14N then writes as ` `), and normalises attribute whitespace — including, now, the collapse for attributes declared non-CDATA; verified by the `c14n` tests | yes | diff --git a/docs/conformance-gaps.md b/docs/conformance-gaps.md index 8b70b605..281400a0 100644 --- a/docs/conformance-gaps.md +++ b/docs/conformance-gaps.md @@ -49,7 +49,7 @@ W3C disagreements: 0 + 0 + 0 + 1 + 8 + 23 + 30 + 31 + 0 = 93. Measured 2026-09-1 -The unit-test suite is 2,941 tests. +The unit-test suite is 2,950 tests. The last two rows are not W3C suites but real-world corpora — DocBook xslTNG's diff --git a/docs/known-gaps.md b/docs/known-gaps.md index 2efbb3d8..cd8f4c54 100644 --- a/docs/known-gaps.md +++ b/docs/known-gaps.md @@ -131,8 +131,14 @@ Found by the third memory audit (2026-11-01) and not fixed, each with the reason (`withSel`, `hostWithCurrent`, the leaf-hook copy), and built-in shallow-copy copies a document-level result twice; removing that copy changes an error case into output. -- **Accumulators keep two per-node maps.** Node-indexed slices need an ordinal - `xdm.Node` does not export. +- **A single text value of a few to tens of MB allocates up to about 1.9× its + size while parsing** (10 MiB: 1.87×; the tree keeps 1.09×; 60 MiB: 1.13×). + The decoder's buffer grows in steps of eight. Sizing it to the input left + in one step reached 1.08×, but a document of eight 3 MiB values went from + 1.33× to 2.04×, so it was reverted. +- **`c14n.BytesNodeSet` on a caller's own `NodeSet` still costs about 2.1× its + output.** Sizing it would call `Contains` twice per node, which the API + promises not to do; the package's `Subtree` and `ExcludeSubtree` are sized. - **XQuery inline functions capture their whole scope.** The XPath and XSLT closure (`InlineFunctionExpr`) keeps only what its body reads; `xquery/inline.go` has its own implementation. diff --git a/docs/options.md b/docs/options.md index 3f0880f3..7701fffa 100644 --- a/docs/options.md +++ b/docs/options.md @@ -704,7 +704,11 @@ evaluation's. A nested evaluation therefore never resets or raises a budget. `WithStaticBaseURI` and `WithStaticHost` set the defaults an expression compiled without them uses (a one-shot `xpath.Eval` compiles at the context's version). A host whose expressions all carry one version or static host sets it once on -the context, so evaluating them does not copy the context to install it. +the context, so evaluating them does not copy the context to install it. This +matters for a host that reuses one context: an expression compiled as XPath 3.1 +evaluated against `NewContext`'s 2.0 default copies the 96-byte context on every +call, about a quarter of a simple filter's allocation per node, until the host +calls `ctx = ctx.WithVersion(xpath.XPath31)` once. `StaticNamespaces()` is installed only for an expression that calls or references `fn:format-date`, `fn:format-dateTime`, `fn:format-time` or `fn:function-lookup`; any other expression sees its caller's (nil at the top diff --git a/docs/stats.md b/docs/stats.md index f32651b2..d8837195 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -23,7 +23,7 @@ claim is only as good as the command behind it. | figure | count | counted by | |---|---:|---| -| Unit tests | 2,941 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | +| Unit tests | 2,950 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Fuzz targets | 12 | `grep -rn '^func Fuzz' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Limit boundary tests | 14 | `grep -hc '^func Test' ./*/limits_boundary*_test.go \| awk '{n += $1} END {print n + 0}'` | diff --git a/docs/testing.md b/docs/testing.md index 8ff8d51e..093e85c3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,7 +21,7 @@ let something through, and the column that matters is the last one. | layer | count | catches | misses | |---|---:|---|---| -| **Unit tests** | 2,941 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** | 2,950 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | | **Limit boundary tests** | 14 tests | an off-by-one or an overflow at the edge of a configurable limit | a limit nobody added to the inventory | | **Race detector** | same tests | shared state a single-goroutine run never reveals | a data race on a path no test walks | | **W3C conformance suites** | 152,241 cases | systematic divergence from the specification | what the suites do not ask about — see below | diff --git a/docs/todo.md b/docs/todo.md index 17727f72..52ce996e 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -23,7 +23,7 @@ Current position: | XSLT 3.0 | 99.80% — 11,495 of 11,518 in scope (23 failing); 3 of those need more of the §19.8 streamability analysis | | RELAX NG | 100.00% — 965 of 965 | | Schemas wrongly refused | 7 — 6 on XSD 1.0, 1 on 1.1 | -| Tests | 2,941 `func Test` declarations, clean under `-race` | +| Tests | 2,950 `func Test` declarations, clean under `-race` | Every one of those failures, and why it is still open, is catalogued in [known-gaps.md](known-gaps.md). This file is the forward-looking half — what