From ac4973e34a88bf8053f13e01d43aa49e6783f6cc Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 13:37:17 -0500 Subject: [PATCH 01/45] xdm: hold namespace nodes weakly so unreferenced ones are freed --- xdm/build.go | 10 +--- xdm/nsbindings_test.go | 4 +- xdm/record.go | 11 ++--- xdm/sideweak.go | 109 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 17 deletions(-) create mode 100644 xdm/sideweak.go diff --git a/xdm/build.go b/xdm/build.go index a1401791..84d19f63 100644 --- a/xdm/build.go +++ b/xdm/build.go @@ -239,14 +239,8 @@ func (t *Tree) truncate(i uint32) { delete(t.source.foreignPos, j) } } - if len(t.side) > 0 { - t.sideMu.Lock() - for k := range t.side { - if k.owner >= i { - delete(t.side, k) - } - } - t.sideMu.Unlock() + if t.side != nil { + t.sideTruncate(i) } t.n = i } diff --git a/xdm/nsbindings_test.go b/xdm/nsbindings_test.go index ee0182e4..3ad0dd18 100644 --- a/xdm/nsbindings_test.go +++ b/xdm/nsbindings_test.go @@ -40,8 +40,8 @@ func TestInScopeNamespaceBindings(t *testing.T) { if k != 1 { t.Errorf("break did not stop the iteration") } - if len(tr.side) != 0 { - t.Errorf("iterating made %d namespace nodes", len(tr.side)) + if tr.side != nil { + t.Errorf("iterating made %d namespace nodes", len(tr.side.m)) } if a := testing.AllocsPerRun(10, func() { for range r.InScopeNamespaceBindings() { diff --git a/xdm/record.go b/xdm/record.go index 70ea8e63..8c45cf1b 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -145,7 +145,7 @@ type Tree struct { // Namespace nodes, made on demand. sideMu sync.Mutex - side map[sideKey]*Node + side *sideTable } // treeSource is the part of a Tree that parsing fills in (see Tree.source). @@ -1470,7 +1470,7 @@ func (n *Node) sideNode(frame, index, owner uint32, slot int) *Node { key := sideKey{n.self, prefix} t.sideMu.Lock() defer t.sideMu.Unlock() - if s, ok := t.side[key]; ok { + if s := t.sideLive(key); s != nil { return s } // The slot places the node among n's others, by prefix, for document @@ -1485,12 +1485,9 @@ func (n *Node) sideNode(frame, index, owner uint32, slot int) *Node { } } s := &Node{tree: t, kind: uint8(KindNamespace), flags: fSide, - self: uint32(len(t.side)), parent: n.self, end: uint32(slot), prev: owner, + self: t.sideSeq(), parent: n.self, end: uint32(slot), prev: owner, v0: frame, v1: index} - if t.side == nil { - t.side = map[sideKey]*Node{} - } - t.side[key] = s + t.sideAdd(key, s) return s } diff --git a/xdm/sideweak.go b/xdm/sideweak.go new file mode 100644 index 00000000..ba171d53 --- /dev/null +++ b/xdm/sideweak.go @@ -0,0 +1,109 @@ +package xdm + +import ( + "runtime" + "weak" +) + +// sideTable is a tree's namespace nodes. They are made on demand and held +// weakly: a node lives while a caller holds it, so pointer identity, Is and +// IdentityKey (which holds the pointer) stay true for every holder, and a +// node nobody holds is freed instead of staying for the life of the tree, and +// its entry is swept after the next collection. seq numbers the nodes in the order they were made. +type sideTable struct { + m map[sideKey]weak.Pointer[Node] + seq uint32 + peak int // most entries held since the map was last rebuilt + armed bool // a sweep is waiting for the next collection +} + +// sideLive returns the live node for key, or nil. Caller holds sideMu. +func (t *Tree) sideLive(key sideKey) *Node { + if t.side == nil { + return nil + } + if w, ok := t.side.m[key]; ok { + return w.Value() + } + return nil +} + +// sideAdd records s under key. Caller holds sideMu. +func (t *Tree) sideAdd(key sideKey, s *Node) { + if t.side == nil { + t.side = &sideTable{m: map[sideKey]weak.Pointer[Node]{}} + } + st := t.side + st.seq++ + st.m[key] = weak.Make(s) + if len(st.m) > st.peak { + st.peak = len(st.m) + } + if !st.armed { + st.armed = true + // A cleanup per node costs a second runtime special each; one per + // garbage collection, while the table is not empty, sweeps the + // entries the collection has cleared. + runtime.AddCleanup(&sideTick{}, sideSweep, t) + } +} + +// sideTick is what the sweep cleanup is attached to: nothing refers to it, +// so its cleanup runs after the next collection. It holds a pointer so the +// allocator does not batch it into a tiny block that outlives the cycle. +type sideTick struct{ _ *int } + +// sideSweep drops the entries whose nodes a collection has freed, and gives +// the memory of a table that has shrunk a lot back (a Go map never does). +func sideSweep(t *Tree) { + t.sideMu.Lock() + defer t.sideMu.Unlock() + st := t.side + if st == nil { + return + } + st.armed = false + for k, w := range st.m { + if w.Value() == nil { + delete(st.m, k) + } + } + switch { + case len(st.m) == 0: + t.side = nil + return + case len(st.m) < st.peak/4: + m := make(map[sideKey]weak.Pointer[Node], len(st.m)) + for k, w := range st.m { + m[k] = w + } + st.m, st.peak = m, len(m) + } + st.armed = true + runtime.AddCleanup(&sideTick{}, sideSweep, t) +} + +// sideSeq is the next sequence number. Caller holds sideMu. +func (t *Tree) sideSeq() uint32 { + if t.side == nil { + return 0 + } + return t.side.seq +} + +// sideTruncate forgets the namespace nodes of elements from index i on. +func (t *Tree) sideTruncate(i uint32) { + t.sideMu.Lock() + defer t.sideMu.Unlock() + if t.side == nil { + return + } + for k := range t.side.m { + if k.owner >= i { + delete(t.side.m, k) + } + } + if len(t.side.m) == 0 { + t.side = nil + } +} From fcab85a054100fc761b7973e97f99d24b59ae723 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 13:38:01 -0500 Subject: [PATCH 02/45] xdm: chunk namespace nodes under one weak handle --- xdm/record.go | 6 ++--- xdm/sideweak.go | 68 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/xdm/record.go b/xdm/record.go index 8c45cf1b..f9b0275e 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1484,11 +1484,9 @@ func (n *Node) sideNode(frame, index, owner uint32, slot int) *Node { } } } - s := &Node{tree: t, kind: uint8(KindNamespace), flags: fSide, + return t.sideAdd(key, Node{tree: t, kind: uint8(KindNamespace), flags: fSide, self: t.sideSeq(), parent: n.self, end: uint32(slot), prev: owner, - v0: frame, v1: index} - t.sideAdd(key, s) - return s + v0: frame, v1: index}) } // --- Identity and order --------------------------------------------------- diff --git a/xdm/sideweak.go b/xdm/sideweak.go index ba171d53..cda6cde5 100644 --- a/xdm/sideweak.go +++ b/xdm/sideweak.go @@ -5,16 +5,35 @@ import ( "weak" ) +// sideChunkLen is how many namespace nodes share one allocation, and so one +// weak handle: the runtime keeps a list per heap span of every object with a +// weak handle, and a handle per node made a namespace-heavy query several +// times slower. +const sideChunkLen = 16 + +type sideChunk [sideChunkLen]Node + +// sideEnt locates a namespace node: slot i of a chunk, held weakly. +type sideEnt struct { + w weak.Pointer[sideChunk] + i uint8 +} + // sideTable is a tree's namespace nodes. They are made on demand and held -// weakly: a node lives while a caller holds it, so pointer identity, Is and -// IdentityKey (which holds the pointer) stay true for every holder, and a -// node nobody holds is freed instead of staying for the life of the tree, and -// its entry is swept after the next collection. seq numbers the nodes in the order they were made. +// weakly, a chunk at a time: a node lives while a caller holds it or any +// node of its chunk, so pointer identity, Is and IdentityKey (which holds the +// pointer) stay true for every holder, and nodes nobody holds are freed +// instead of staying for the life of the tree. Their entries are swept after +// the next collection. seq numbers the nodes in the order they were made. type sideTable struct { - m map[sideKey]weak.Pointer[Node] + m map[sideKey]sideEnt seq uint32 peak int // most entries held since the map was last rebuilt armed bool // a sweep is waiting for the next collection + + cur *sideChunk // being filled; held strongly until the next sweep + used int + curW weak.Pointer[sideChunk] } // sideLive returns the live node for key, or nil. Caller holds sideMu. @@ -22,30 +41,40 @@ func (t *Tree) sideLive(key sideKey) *Node { if t.side == nil { return nil } - if w, ok := t.side.m[key]; ok { - return w.Value() + if e, ok := t.side.m[key]; ok { + if c := e.w.Value(); c != nil { + return &c[e.i] + } } return nil } -// sideAdd records s under key. Caller holds sideMu. -func (t *Tree) sideAdd(key sideKey, s *Node) { +// sideAdd makes the node proto describes and records it under key. Caller +// holds sideMu. +func (t *Tree) sideAdd(key sideKey, proto Node) *Node { if t.side == nil { - t.side = &sideTable{m: map[sideKey]weak.Pointer[Node]{}} + t.side = &sideTable{m: map[sideKey]sideEnt{}} } st := t.side + if st.cur == nil || st.used == sideChunkLen { + st.cur, st.used = new(sideChunk), 0 + st.curW = weak.Make(st.cur) + } + s := &st.cur[st.used] + *s = proto + st.m[key] = sideEnt{st.curW, uint8(st.used)} + st.used++ st.seq++ - st.m[key] = weak.Make(s) if len(st.m) > st.peak { st.peak = len(st.m) } if !st.armed { st.armed = true - // A cleanup per node costs a second runtime special each; one per - // garbage collection, while the table is not empty, sweeps the - // entries the collection has cleared. + // One cleanup per garbage collection, while the table is not + // empty, sweeps the entries the collection has cleared. runtime.AddCleanup(&sideTick{}, sideSweep, t) } + return s } // sideTick is what the sweep cleanup is attached to: nothing refers to it, @@ -63,8 +92,9 @@ func sideSweep(t *Tree) { return } st.armed = false - for k, w := range st.m { - if w.Value() == nil { + st.cur = nil // else its entries would never clear + for k, e := range st.m { + if e.w.Value() == nil { delete(st.m, k) } } @@ -73,9 +103,9 @@ func sideSweep(t *Tree) { t.side = nil return case len(st.m) < st.peak/4: - m := make(map[sideKey]weak.Pointer[Node], len(st.m)) - for k, w := range st.m { - m[k] = w + m := make(map[sideKey]sideEnt, len(st.m)) + for k, e := range st.m { + m[k] = e } st.m, st.peak = m, len(m) } From e1b5e1f301878b3e909d5ec67947d5da317e42cd Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 13:39:24 -0500 Subject: [PATCH 03/45] xpath, xslt: pin namespace node identity across collections --- xpath/nsweak_test.go | 165 +++++++++++++++++++++++++++++++++++++++++++ xslt/nsweak_test.go | 38 ++++++++++ 2 files changed, 203 insertions(+) create mode 100644 xpath/nsweak_test.go create mode 100644 xslt/nsweak_test.go diff --git a/xpath/nsweak_test.go b/xpath/nsweak_test.go new file mode 100644 index 00000000..dd8b5ff4 --- /dev/null +++ b/xpath/nsweak_test.go @@ -0,0 +1,165 @@ +package xpath + +import ( + "runtime" + "testing" + "time" + + "github.com/knroy/go-xml/v2/xdm" +) + +func gid(n *xdm.Node) string { + s, _ := MustCompile("generate-id(.)", nil).EvalString(NewContext(n, Builtins())) + return s +} + +// settle runs enough collections, and waits long enough for the cleanup that +// sweeps the namespace-node table, that anything unreferenced is gone. +func settle() { + for i := 0; i < 3; i++ { + runtime.GC() + time.Sleep(20 * time.Millisecond) + } +} + +// TestNamespaceNodeIdentityAcrossGC pins what holding namespace nodes weakly +// must keep: while a caller holds a namespace node, every later request for +// it returns that same node, whatever collections ran in between, so is, +// deep-equal, generate-id, union de-duplication and use as a Go map key agree +// between two evaluations that both hold it. +func TestNamespaceNodeIdentityAcrossGC(t *testing.T) { + const doc = `` + tree, err := xdm.ParseString(doc, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + root := tree.Root.FirstChild() + eval := func(expr string, vars map[string]xdm.Sequence) xdm.Sequence { + t.Helper() + ctx := NewContext(tree.Root, Builtins()).WithVersion(XPath31) + for k, v := range vars { + ctx = ctx.WithVar(xdm.QName{Local: k}, v) + } + seq, err := Eval(expr, ctx, nil) + if err != nil { + t.Fatalf("%s: %v", expr, err) + } + return seq + } + ebv := func(expr string, vars map[string]xdm.Sequence) bool { + t.Helper() + b, err := EffectiveBooleanValue(eval(expr, vars)) + if err != nil { + t.Fatal(err) + } + return b + } + + held := NamespaceNodesOf(root.FirstChild()) // c's axis: a, b, xml + if len(held) != 3 { + t.Fatalf("got %d namespace nodes, want 3", len(held)) + } + ids := make([]string, len(held)) + keys := map[xdm.IdentityKey]int{} + byPtr := map[*xdm.Node]int{} + for i, n := range held { + s := gid(n) + ids[i] = s + keys[n.Identity()] = i + byPtr[n] = i + } + heldSeq := make(xdm.Sequence, len(held)) + for i, n := range held { + heldSeq[i] = n + } + + // Other namespace nodes come and go while these are held. + for i := 0; i < 4; i++ { + eval("count(//namespace::*)", nil) + settle() + } + + again := NamespaceNodesOf(root.FirstChild()) + for i, n := range again { + if n != held[i] || !n.Is(held[i]) || n.Identity() != held[i].Identity() { + t.Errorf("node %d: not the node still held", i) + } + if keys[n.Identity()] != i || byPtr[n] != i { + t.Errorf("node %d: map key lookup differs", i) + } + s := gid(n) + if s != ids[i] { + t.Errorf("node %d: generate-id %q, was %q", i, s, ids[i]) + } + } + + vars := map[string]xdm.Sequence{"held": heldSeq} + for _, tc := range []struct { + expr string + want bool + }{ + {`/r/c/namespace::b is $held[2]`, true}, + {`/r/c/namespace::a is $held[1]`, true}, + {`/r/namespace::a is $held[1]`, false}, + {`deep-equal(/r/c/namespace::*, $held)`, true}, + {`count(/r/c/namespace::* | $held) = 3`, true}, + {`count(/r/c/namespace::* intersect $held) = 3`, true}, + {`count(/r/c/namespace::* except $held) = 0`, true}, + {`count(distinct-values(($held, /r/c/namespace::*)!string())) = 3`, true}, + {`generate-id(/r/c/namespace::b) = generate-id($held[2])`, true}, + {`/r/c/namespace::b << /r/c/namespace::a`, false}, + {`/r/c/namespace::a << /r/c/namespace::b`, true}, + } { + if got := ebv(tc.expr, vars); got != tc.want { + t.Errorf("%s = %v, want %v", tc.expr, got, tc.want) + } + } + runtime.KeepAlive(held) + runtime.KeepAlive(tree) +} + +// TestNamespaceNodesAreReleased checks the point of holding them weakly: a +// query that touches many namespace nodes and keeps none leaves nothing +// behind, and a node made again after its predecessor was freed is +// indistinguishable by every observable. +func TestNamespaceNodesAreReleased(t *testing.T) { + var b []byte + b = append(b, ``...) + for i := 0; i < 2000; i++ { + b = append(b, ``...) + } + b = append(b, ``...) + tree, err := xdm.ParseString(string(b), xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + probe := func() (id string, order int) { + e := tree.Root.FirstChild().FirstChild() + n := NamespaceNodesOf(e)[1] + s := gid(n) + return s, n.Order() + } + id0, ord0 := probe() + + var m runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&m) + before := m.HeapAlloc + for i := 0; i < 3; i++ { + seq, err := Eval("count(//namespace::*)", NewContext(tree.Root, Builtins()), nil) + if err != nil || len(seq) != 1 { + t.Fatal(err) + } + } + settle() + runtime.ReadMemStats(&m) + if grown := int64(m.HeapAlloc) - int64(before); grown > 64<<10 { + t.Errorf("heap grew %d B after a query that kept no namespace node", grown) + } + + id1, ord1 := probe() + if id0 != id1 || ord0 != ord1 { + t.Errorf("remade node: generate-id %q order %d, was %q order %d", id1, ord1, id0, ord0) + } + runtime.KeepAlive(tree) +} diff --git a/xslt/nsweak_test.go b/xslt/nsweak_test.go new file mode 100644 index 00000000..dea21007 --- /dev/null +++ b/xslt/nsweak_test.go @@ -0,0 +1,38 @@ +package xslt_test + +import ( + "runtime" + "strings" + "testing" + "time" +) + +// TestKeyOverNamespaceNodesAcrossGC checks xsl:key over namespace nodes, whose +// index walk (xpath.NamespaceNodesOf) makes them: they are held weakly, so +// the index must keep them, and a lookup made after collections must still +// find the node the index holds. +func TestKeyOverNamespaceNodesAcrossGC(t *testing.T) { + const sheet = ` + + + + + + + + + + ; + + +` + const doc = `` + got := transformTo(t, sheet, doc) + runtime.GC() + time.Sleep(20 * time.Millisecond) + runtime.GC() + // urn:a is in scope on r and both c elements. + if want := strings.Repeat("3truetruetrue;", 3); !strings.HasSuffix(got, want) { + t.Errorf("got %q, want %q", got, want) + } +} From b4b605284319e6c20dd9b91a36b9fe7d3597f8ac Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 13:43:58 -0500 Subject: [PATCH 04/45] xdm: ParseBursts parses a document one selected subtree at a time --- xdm/bursts.go | 180 +++++++++++++++++++++++++++++++++++++++++++++ xdm/bursts_test.go | 142 +++++++++++++++++++++++++++++++++++ xdm/parse.go | 20 +++++ 3 files changed, 342 insertions(+) create mode 100644 xdm/bursts.go create mode 100644 xdm/bursts_test.go diff --git a/xdm/bursts.go b/xdm/bursts.go new file mode 100644 index 00000000..42ceca84 --- /dev/null +++ b/xdm/bursts.go @@ -0,0 +1,180 @@ +package xdm + +import "io" + +// Burst-mode parsing: the incremental half of streamed XSLT execution. +// +// ParseBursts reads a document exactly as Parse does -- the same tokeniser, +// the same well-formedness checks, the same DOCTYPE handling, and the same +// MaxBytes, MaxNodes and MaxDepth, counted over the whole document -- but it +// never holds the whole tree. Only the chain of open ancestors (the "spine") +// is kept. Each element at the end of path is handed to fn once it is +// complete, as the document element's descendant in a small tree of its own, +// and is then dropped, records and text both. Everything else -- elements off +// the path, and the text, comments and processing instructions directly +// inside a spine element -- is built, checked, and dropped the same way. +// +// The tree fn receives holds the document node (with the document's base and +// document URIs), a shallow copy of each ancestor (name, namespace +// declarations, attributes, base URI) and a deep copy of the element. So the +// element's base URI, in-scope namespaces, attributes, parent and ancestors +// read as they would in the whole tree; its siblings, and anything else +// outside its ancestry, do not exist there. Each burst is a new tree, so +// nodes from different bursts are distinct and order by creation, which is +// document order; a node fn keeps stays valid after fn returns. + +// ParseBursts parses r and calls fn with each element whose ancestry of +// element names, from the document element down, is path. An error from fn +// stops the parse and is returned. +func ParseBursts(r io.Reader, opts ParseOptions, path []QName, fn func(*Node) error) error { + if len(path) == 0 { + panic("xdm: ParseBursts needs a non-empty path") + } + opts.bursts = &burster{path: path, fn: fn} + _, err := parse(r, "", opts) + return err +} + +// burster is ParseBursts' state inside parse. +type burster struct { + path []QName + fn func(*Node) error + // spine is how many levels of the open chain match path and are kept. + spine int + // drop is the depth of the element being built only to be dropped (a + // burst, or an element off the path), or 0. + drop int + // marks[d] is the text store's state once the spine element at depth d + // (0: the document node) had stored its attributes. Everything stored + // after it belongs to content that is dropped. + marks []textMark + v11 bool +} + +// start classifies the element just built at depth d and opened as el. +func (b *burster) start(el *Node, d int) { + if b.drop != 0 { + return + } + if d == b.spine+1 && b.matches(el, d) { + if d < len(b.path) { + b.spine = d + b.marks = append(b.marks, el.tree.text.mark()) + return + } + } + b.drop = d +} + +func (b *burster) matches(el *Node, d int) bool { + n := el.Name() + return n.URI == b.path[d-1].URI && n.Local == b.path[d-1].Local +} + +// end handles the close of el, at depth d, whose parent is now current. It +// delivers a burst and drops whatever is not spine. +func (b *burster) end(el *Node, d int, spaces *spaceTable) error { + switch { + case b.drop == d: + b.drop = 0 + if d == len(b.path) && d == b.spine+1 && b.matches(el, d) { + if err := b.deliver(el); err != nil { + return err + } + } + case b.drop == 0 && d == b.spine && d > 0: + b.spine-- + b.marks = b.marks[:len(b.marks)-1] + default: + return nil + } + b.discardLast(el.Parent(), spaces) + return nil +} + +// trimLeaves drops the leaves completed directly inside the spine element +// cur, at depth d: text, comments and processing instructions no path step +// selects. +func (b *burster) trimLeaves(cur *Node, d int, spaces *spaceTable) { + if b.drop != 0 || d == 0 || d != b.spine { + return + } + for last := cur.LastChild(); last != nil && last.isLeaf(); last = cur.LastChild() { + b.discardLast(cur, spaces) + } +} + +// discardLast removes p's last child and the text stored since p's spine mark. +func (b *burster) discardLast(p *Node, spaces *spaceTable) { + p.RemoveLastChild() + p.tree.text.rewind(b.marks[len(b.marks)-1]) + // The whitespace table remembers ranges in the store, and some of them + // were just rewound. + *spaces = spaceTable{} +} + +// deliver copies el and its ancestry into a tree of its own and hands it to +// fn. +func (b *burster) deliver(el *Node) error { + src := el.tree + t := NewTree() + t.XMLVersion = "1.0" + if b.v11 { + t.XMLVersion = "1.1" + } + t.Root.SetBaseURI(src.Root.BaseURI()) + t.Root.SetDocumentURI(src.Root.DocumentURI()) + var chain []*Node + for a := el.Parent(); a != nil && a.Kind() == KindElement; a = a.Parent() { + chain = append(chain, a) + } + p := t.Root + for i := len(chain) - 1; i >= 0; i-- { + a := chain[i] + c := p.AppendShallowCopy(a) + for prefix, uri := range a.DeclaredNamespaces() { + c.AddNamespace(prefix, uri) + } + for at := range a.Attrs() { + c.AppendAttr(at.Name(), at.Value()).CopyTypingFrom(at) + } + p = c + } + c := p.AppendCopy(el) + t.Finalize() + return b.fn(c) +} + +// textMark is a point in a textStore that rewind returns it to. +type textMark struct { + blocks int + cur int32 + size int32 + curLen int + hasBlock bool +} + +func (s *textStore) mark() textMark { + m := textMark{blocks: len(s.blocks), cur: s.cur, size: s.size} + if s.cur > 0 { + m.curLen = len(s.blocks[s.cur-1]) + m.hasBlock = true + } + return m +} + +// rewind forgets every value stored since m. Nothing may still read one of +// them: ParseBursts only rewinds over records it has removed, and the trees it +// hands out copy their values (a value long enough to be aliased rather than +// copied lives in a block of its own, which rewind releases and never +// rewrites). +func (s *textStore) rewind(m textMark) { + for i := m.blocks; i < len(s.blocks); i++ { + s.blocks[i] = nil + } + s.blocks = s.blocks[:m.blocks] + s.cur, s.size = m.cur, m.size + if m.hasBlock { + s.blocks[s.cur-1] = s.blocks[s.cur-1][:m.curLen] + } +} diff --git a/xdm/bursts_test.go b/xdm/bursts_test.go new file mode 100644 index 00000000..08c7e097 --- /dev/null +++ b/xdm/bursts_test.go @@ -0,0 +1,142 @@ +package xdm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// burstDoc exercises what a burst must carry over from the whole tree: an +// inherited namespace and xml:base, attribute defaults from the DOCTYPE, an +// entity, text long enough to get a text block of its own, and spine content +// (text, comments, PIs, off-path elements) that must be dropped. +func burstDoc(n int) string { + var b strings.Builder + b.WriteString(`]>` + + `x` + + `not me`) + for i := range n { + fmt.Fprintf(&b, "\n &e;%d", i, i) + if i%7 == 3 { + b.WriteString(strings.Repeat("long ", 3000)) + } + b.WriteString("text") + } + b.WriteString("") + return b.String() +} + +func TestParseBurstsMatchesTree(t *testing.T) { + src := burstDoc(40) + opts := ParseOptions{AllowDOCTYPE: true, BaseURI: "http://ex/d.xml", DocumentURI: "http://ex/d.xml"} + tree, err := ParseString(src, opts) + if err != nil { + t.Fatal(err) + } + var want []string + for a := range tree.Root.Children() { + if a.Kind() != KindElement { + continue + } + for b := range a.Children() { + if b.Kind() != KindElement || b.Name().Local != "b" { + continue + } + for r := range b.Children() { + if r.Kind() == KindElement && r.Name().Local == "r" { + want = append(want, dump(r)+ancestry(r)) + } + } + } + } + path := []QName{{URI: "urn:a", Local: "a"}, {URI: "urn:a", Local: "b"}, {URI: "urn:a", Local: "r"}} + var got []*Node + err = ParseBursts(strings.NewReader(src), opts, path, func(r *Node) error { + got = append(got, r) + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(got) != len(want) { + t.Fatalf("got %d bursts, want %d", len(got), len(want)) + } + // Compared after the parse: a kept burst must survive the rest of it. + for i, r := range got { + if g := dump(r) + ancestry(r); g != want[i] { + t.Fatalf("burst %d differs\nwant\n%s\ngot\n%s", i, want[i], g) + } + if i > 0 && got[i-1].Tree().ident() >= r.Tree().ident() { + t.Fatalf("burst %d does not order after burst %d", i, i-1) + } + } +} + +func ancestry(n *Node) string { + var b strings.Builder + for a := n.Parent(); a != nil; a = a.Parent() { + fmt.Fprintf(&b, "^%v %q base=%q doc=%q ns=%v\n", a.Name(), a.Value(), a.BaseURI(), a.DocumentURI(), + a.InScopeNamespaces()) + for at := range a.Attrs() { + fmt.Fprintf(&b, " @%v=%q\n", at.Name(), at.Value()) + } + } + return b.String() +} + +// The limits are the document's, not each burst's. +func TestParseBurstsLimitsSpanTheStream(t *testing.T) { + src := "" + strings.Repeat("", 100) + "" + path := []QName{{Local: "a"}, {Local: "r"}} + n := 0 + err := ParseBursts(strings.NewReader(src), ParseOptions{MaxNodes: 50}, path, func(*Node) error { n++; return nil }) + if !errors.Is(err, ErrResourceLimit) { + t.Fatalf("MaxNodes over the stream: got %v after %d bursts", err, n) + } + err = ParseBursts(strings.NewReader(src), ParseOptions{MaxBytes: 200}, path, func(*Node) error { return nil }) + if err == nil { + t.Fatal("MaxBytes over the stream: no error") + } + // Malformed after the bursts is still an error. + err = ParseBursts(strings.NewReader(""), ParseOptions{}, path, func(*Node) error { return nil }) + if err == nil { + t.Fatal("unbalanced end tag after bursts: no error") + } + stop := errors.New("stop") + err = ParseBursts(strings.NewReader(src), ParseOptions{}, path, func(*Node) error { return stop }) + if err != stop { + t.Fatalf("fn's error: got %v", err) + } +} + +// The spine does not grow: what a burst and the text between bursts leave +// behind in the parse's own tree is taken back. +func TestParseBurstsSpineStaysSmall(t *testing.T) { + var b strings.Builder + b.WriteString("") + for i := range 20000 { + fmt.Fprintf(&b, "%s\n", i, strings.Repeat("v", i%50)) + } + b.WriteString("") + n := 0 + opts := ParseOptions{bursts: &burster{ + path: []QName{{Local: "a"}, {Local: "b"}, {Local: "r"}}, + fn: func(*Node) error { n++; return nil }, + }} + tree, err := parse(strings.NewReader(b.String()), "", opts) + if err != nil { + t.Fatal(err) + } + if n != 20000 { + t.Fatalf("%d bursts", n) + } + // Left behind: the document node, and the bytes of nothing. + var bytes int + for _, blk := range tree.text.blocks { + bytes += len(blk) + } + if tree.n > 2 || bytes > 0 { + t.Fatalf("the spine kept %d records and %d text bytes", tree.n, bytes) + } +} diff --git a/xdm/parse.go b/xdm/parse.go index 2ae9a931..7b276ac5 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -126,6 +126,9 @@ type ParseOptions struct { // It is unexported for the same reason as the two fields above: it // describes this parse's place in a larger document, not a knob. entityBudget *entityBudget + + // bursts, when set, makes this parse ParseBursts: see bursts.go. + bursts *burster } // Limits applied when the corresponding ParseOptions field is zero. @@ -312,6 +315,7 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { var ns nsScope var names nameCache depth := 0 + bs := opts.bursts sawRoot := false sawDecl := false standalone := false // the XML declaration said standalone="yes" @@ -365,6 +369,9 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { // tag, a new sibling -- can look at it. if _, text := tok.(*xml.CharData); !text { run.flush(&spaces, stripSpaceAt) + if bs != nil { + bs.trimLeaves(cur, depth, &spaces) + } } // Each token points into the decoder and is copied out here; it is @@ -411,6 +418,10 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { if err := validateStartElement(t, &ns, dec.IsVersion11()); err != nil { return nil, err } + if bs != nil && cur == tree.Root { + bs.marks = append(bs.marks[:0], tree.text.mark()) + bs.v11 = dec.IsVersion11() + } el, decls := buildElement(t, cur, &ns, &names) if off := encodeOffset(start, trackPos); off != 0 { el.setOffset(off) @@ -444,6 +455,9 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { maxNodes, ErrResourceLimit) } cur = el + if bs != nil { + bs.start(el, depth) + } case *xml.EndElement: t := *tok @@ -465,8 +479,14 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { // must not be gated on a strip-space declaration existing. tree.closeLast() ns.pop(len(cur.frame())) + closed := cur cur = cur.Parent() depth-- + if bs != nil { + if err := bs.end(closed, depth+1, &spaces); err != nil { + return nil, err + } + } case *xml.CharData: t := *tok From 302c9320f24356c4ff25de49804dd3196c215fbd Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 13:50:38 -0500 Subject: [PATCH 05/45] xslt: build a rule-less shallow-copy document in place --- xdmbuild/builder.go | 23 ++++++++++++++++++ xdmbuild/inplacedoc_test.go | 31 ++++++++++++++++++++++++ xslt/apply.go | 34 ++++++++++++++++++++------ xslt/shallowdoc_test.go | 48 +++++++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 xdmbuild/inplacedoc_test.go create mode 100644 xslt/shallowdoc_test.go diff --git a/xdmbuild/builder.go b/xdmbuild/builder.go index 3faf4c8e..5c7a60ee 100644 --- a/xdmbuild/builder.go +++ b/xdmbuild/builder.go @@ -134,6 +134,19 @@ func New(p Policy) *Builder { return &Builder{policy: p, refused: new(error)} } +// InPlaceDocument returns a builder whose content is built in place as the children +// of a new document node, which it also returns. It saves the second copy that +// building a sequence and wrapping it with ToDocument costs, for a caller whose +// content cannot be an attribute or a namespace node. Should one arrive anyway +// it raises FaultAttrOnDocument, exactly as ToDocument does, rather than +// attaching it to the document node. +func InPlaceDocument(p Policy) (*Builder, *xdm.Node) { + b := New(p) + t := xdm.NewTree() + b.open = t.Root + return b, t.Root +} + // AppendNode adds a node to the current output position. // // A node that already belongs to a tree is copied first. AppendChild rewrites @@ -496,6 +509,11 @@ func (b *Builder) AddAttributeTyped(name xdm.QName, value string, func (b *Builder) AddAttributeWithTyping(name xdm.QName, value string, typing xdm.Typing) error { + if b.open != nil && b.open.Kind() == xdm.KindDocument { + return b.policy.Err(FaultAttrOnDocument, + fmt.Sprintf("an attribute node (%s) cannot be the content of "+ + "a document node", name.Lexical())) + } if b.open == nil { // A parentless attribute is a legal item in the data model, and a // sequence constructor may produce one: xsl:function as="attribute()" @@ -661,6 +679,11 @@ func freshPrefixOn(el *xdm.Node, want string) string { // add a binding to the element being built, and a namespace node appended as // if it were a child would silently vanish from the result. func (b *Builder) AddNamespace(prefix, uri string) error { + if b.open != nil && b.open.Kind() == xdm.KindDocument { + return b.policy.Err(FaultAttrOnDocument, + fmt.Sprintf("a namespace node (%s) cannot be the content of "+ + "a document node", prefix)) + } if b.open == nil { if !b.countNodes(1) { return b.Refused() diff --git a/xdmbuild/inplacedoc_test.go b/xdmbuild/inplacedoc_test.go new file mode 100644 index 00000000..e5ea620d --- /dev/null +++ b/xdmbuild/inplacedoc_test.go @@ -0,0 +1,31 @@ +package xdmbuild_test + +import ( + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" + "github.com/knroy/go-xml/v2/xdmbuild" +) + +// A builder made by InPlaceDocument builds children in place and refuses an +// attribute or namespace node the way ToDocument does. +func TestInPlaceDocument(t *testing.T) { + b, root := xdmbuild.InPlaceDocument(xsltLike{}) + e := b.StartElement(xdm.QName{Local: "e"}) + e.AppendText("x") + if root.FirstChild() == nil || root.FirstChild().Name().Local != "e" { + t.Fatal("child not built in place") + } + err := b.AddAttribute(xdm.QName{Local: "a"}, "v") + want := "XSLT-" + string(rune('0'+int(xdmbuild.FaultAttrOnDocument))) + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("attribute: got %v, want %s", err, want) + } + if err := b.AddNamespace("p", "urn:p"); err == nil { + t.Error("namespace on document accepted") + } + if root.NumAttrs() != 0 { + t.Error("attribute attached to the document node") + } +} diff --git a/xslt/apply.go b/xslt/apply.go index 74eb46a1..22483ca9 100644 --- a/xslt/apply.go +++ b/xslt/apply.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/knroy/go-xml/v2/xdm" + "github.com/knroy/go-xml/v2/xdmbuild" "github.com/knroy/go-xml/v2/xpath" ) @@ -1227,13 +1228,32 @@ func builtInShallowCopy(rt *runtime, node *xdm.Node, mode string, // children flatten into the parent either way, but a function // declared as="document-node()" gets its result rejected — which is // exactly what merge-096 does. - sub := newOutputBuilder(rt) - if err := builtInDescend(rt, node, mode, params, tunnels, sub, false); err != nil { - return err - } - doc, err := sub.ToDocument() - if err != nil { - return err + var doc *xdm.Node + if mr := rt.sheet.rules[mode]; mr != nil && mr.empty { + // No template rule is in the mode, so only the built-in rules + // run and none of them yields an attribute or a namespace node + // for the document: build the children in place instead of + // building them and copying them into a document afterwards, + // which held the result twice. Should one ever arrive, the + // builder raises XTDE0420 as ToDocument does. + sub, root := xdmbuild.InPlaceDocument(xsltPolicy{nodes: rt.nodeBudget()}) + if err := builtInDescend(rt, node, mode, params, tunnels, sub, false); err != nil { + return err + } + if err := sub.Refused(); err != nil { + return err + } + root.Tree().Finalize() + doc = root + } else { + sub := newOutputBuilder(rt) + if err := builtInDescend(rt, node, mode, params, tunnels, sub, false); err != nil { + return err + } + var err error + if doc, err = sub.ToDocument(); err != nil { + return err + } } // The DOCTYPE and the base URI travel with the copy for the same // reason they do in xsl:copy: the unparsed entity declarations and diff --git a/xslt/shallowdoc_test.go b/xslt/shallowdoc_test.go new file mode 100644 index 00000000..7a871a1c --- /dev/null +++ b/xslt/shallowdoc_test.go @@ -0,0 +1,48 @@ +package xslt + +import ( + "strings" + "testing" +) + +// shallowDocSheet applies a shallow-copy mode to the document node inside a +// function declared as="document-node()", and reports what the copy carries. +// extra is added to the stylesheet; a rule that matches nothing in the source +// puts the mode on the path that builds a sequence and wraps it in a document. +func shallowDocSheet(extra string) string { + return ` + +` + extra + ` + + + + +| +` +} + +const shallowDocSrc = `` + + ` t z ` + +// The in-place build of a rule-less mode must give what the build-then-wrap +// path gives, down to the base URIs. +func TestShallowCopyDocumentInPlaceMatchesWrapped(t *testing.T) { + want := run(t, shallowDocSheet(``), shallowDocSrc) + got := run(t, shallowDocSheet(``), shallowDocSrc) + if got != want { + t.Errorf("in place:\n%s\nwrapped:\n%s", got, want) + } + if !strings.Contains(got, `z`) { + t.Errorf("copy lost content: %s", got) + } +} + +// An attribute produced for the content of a copied document node is still +// XTDE0420; it must not become an attribute of an enclosing element. +func TestShallowCopyDocumentAttributeStillFaults(t *testing.T) { + sheet := shallowDocSheet(`1`) + if _, err := runErr(t, sheet, ``); err == nil || !strings.Contains(err.Error(), "XTDE0420") { + t.Errorf("got %v, want XTDE0420", err) + } +} From c60f8f3fbaf50a94b3d3895fe4220932e28c7552 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 13:52:52 -0500 Subject: [PATCH 06/45] xdm: share whitespace-only text values in built trees --- xdm/record.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/xdm/record.go b/xdm/record.go index 70ea8e63..f0b0e285 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1681,6 +1681,9 @@ type textStore struct { // shared maps a string registered by share to where it is stored, so that // add hands the same range to every value that is that string. shared map[sharedKey][2]uint32 + // ws shares one stored value among the whitespace-only values add is + // given, as a parse does; made on the first. + ws *spaceTable } // sharedKey identifies a string by where it lives: the same storage is the @@ -1716,6 +1719,14 @@ func (s *textStore) add(v string) (uint32, uint32) { if v == "" { return 0, 0 } + if len(v) <= maxSpaceLen { + if b := unsafe.Slice(unsafe.StringData(v), len(v)); onlySpace(b) { + if s.ws == nil { + s.ws = new(spaceTable) + } + return s.ws.text(s, b) + } + } if s.shared != nil && len(v) >= shareMin { if r, ok := s.shared[sharedKey{unsafe.StringData(v), len(v)}]; ok { return r[0], r[1] From 3f185febf766b9f09ae0d7336936b2be57f6eecc Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 13:53:21 -0500 Subject: [PATCH 07/45] xslt: run the narrowest streamable xsl:source-document in bursts --- xslt/compile.go | 3 + xslt/compile_instr.go | 54 ++- xslt/instructions.go | 10 + xslt/resolver.go | 28 +- xslt/runtime.go | 4 + xslt/sourcedoc.go | 51 ++- xslt/streamburst.go | 737 +++++++++++++++++++++++++++++++++++++++ xslt/streamburst_test.go | 140 ++++++++ 8 files changed, 994 insertions(+), 33 deletions(-) create mode 100644 xslt/streamburst.go create mode 100644 xslt/streamburst_test.go diff --git a/xslt/compile.go b/xslt/compile.go index 923d52a4..a3a62047 100644 --- a/xslt/compile.go +++ b/xslt/compile.go @@ -12,6 +12,9 @@ import ( // compiler holds state while compiling a stylesheet and its included modules. type compiler struct { + // burst is the burst plan of the streamable xsl:source-document being + // compiled, which records the compiled form of its loop. + burst *burstPlan opts CompileOptions sheet *Stylesheet declOrder int diff --git a/xslt/compile_instr.go b/xslt/compile_instr.go index 29c32d04..73756be5 100644 --- a/xslt/compile_instr.go +++ b/xslt/compile_instr.go @@ -971,7 +971,9 @@ func (c *compiler) compileForEach(n *xdm.Node, ns xpath.NamespaceResolver) (Inst if err != nil { return nil, err } - return &forEachInstr{sel: sel, sorts: sorts, body: body}, nil + instr := &forEachInstr{sel: sel, sorts: sorts, body: body} + c.noteBurstLoop(n, instr, sel) + return instr, nil } func (c *compiler) compileIf(n *xdm.Node, ns xpath.NamespaceResolver) (Instruction, error) { @@ -2024,6 +2026,9 @@ type iterateInstr struct { // from a body compiled here. type breakSignal struct{} +// errStopIterate ends iterateInstr's loop after xsl:break. +var errStopIterate = errors.New("xsl:iterate stopped") + func (breakSignal) Error() string { return "xsl:break" } // nextIterationSignal unwinds to the end of one turn of the loop, carrying the @@ -2113,9 +2118,13 @@ func appendSequence(seq xdm.Sequence, out *outputBuilder) error { } func (i *iterateInstr) Execute(rt *runtime, out *outputBuilder) error { - seq, err := i.sel.Eval(rt.ctx) - if err != nil { - return err + feed := rt.burstsFor(i) + var seq xdm.Sequence + if feed == nil { + var err error + if seq, err = i.sel.Eval(rt.ctx); err != nil { + return err + } } // 6.8: the current template rule is cleared within xsl:iterate. rt = rt.clearCurrentRule() @@ -2133,8 +2142,9 @@ func (i *iterateInstr) Execute(rt *runtime, out *outputBuilder) error { carried[k] = v } broke := false - for idx, it := range seq { - sub := rt.withCurrent(it, idx+1, len(seq)) + // step runs one turn; errStopIterate ends the loop. + step := func(pos int, it xdm.Item, size int) error { + sub := rt.withCurrent(it, pos, size) mark := sub.localsMark() for k, p := range i.params { sub = sub.bindLocal(p.Name, carried[k]) @@ -2144,7 +2154,7 @@ func (i *iterateInstr) Execute(rt *runtime, out *outputBuilder) error { if err != nil { if _, ok := err.(breakSignal); ok { broke = true - break + return errStopIterate } if nx, ok := err.(nextIterationSignal); ok { // Only the parameters xsl:next-iteration named change; the @@ -2176,10 +2186,37 @@ func (i *iterateInstr) Execute(rt *runtime, out *outputBuilder) error { carried[k] = conv } } - continue + return nil } return err } + return nil + } + if feed != nil { + // After xsl:break the rest of the document is still read, so that a + // malformed tail is the error it is when the tree is built first. + pos := 0 + err := feed.read(func(it xdm.Item) error { + if broke { + return nil + } + pos++ + if err := step(pos, it, 0); err != errStopIterate { + return err + } + return nil + }) + if err != nil { + return err + } + } else { + for idx, it := range seq { + if err := step(idx+1, it, len(seq)); err == errStopIterate { + break + } else if err != nil { + return err + } + } } // "It is not evaluated if the evaluation is terminated using xsl:break." @@ -2260,6 +2297,7 @@ func (c *compiler) compileIterate(n *xdm.Node, ns *nsResolver) (Instruction, err return nil, err } instr.body = body + c.noteBurstLoop(n, instr, sel) return instr, nil } diff --git a/xslt/instructions.go b/xslt/instructions.go index 6a160edd..8b54cef8 100644 --- a/xslt/instructions.go +++ b/xslt/instructions.go @@ -1664,6 +1664,16 @@ type forEachInstr struct { } func (i *forEachInstr) Execute(rt *runtime, out *outputBuilder) error { + if f := rt.burstsFor(i); f != nil { + pos := 0 + return f.read(func(it xdm.Item) error { + if err := rt.ctx.Err(); err != nil { + return err + } + pos++ + return execSequence(i.body, rt.withCurrent(it, pos, 0).clearCurrentRule(), out) + }) + } seq, err := i.sel.Eval(rt.ctx) if err != nil { return err diff --git a/xslt/resolver.go b/xslt/resolver.go index af1b301c..f5a9889f 100644 --- a/xslt/resolver.go +++ b/xslt/resolver.go @@ -689,17 +689,8 @@ func (r *FileResolver) publish(path string, tree *xdm.Tree, size int64) { r.cacheBytes += size } -// parseUncached reads and parses a file. It holds no lock: this is the work -// that used to run under r.mu. -func (r *FileResolver) parseUncached( - path string, trackPos bool, b *xdm.EntityBudget) (*xdm.Tree, int64, error) { - // A file: URI rather than the bare path. This base URI is stamped on - // every element of the module and is what fn:document, fn:resolve-uri - // and fn:static-base-uri resolve against, and those are defined over - // URIs: an absolute filesystem path has no scheme, so it is still a - // *relative* URI reference and resolving against it drops everything - // before the last separator. resolvePath strips the scheme back off, so - // the filesystem sees the same path either way. +// parseOptions are the options a document at path is parsed with. +func (r *FileResolver) parseOptions(path string, trackPos bool) xdm.ParseOptions { opts := xdm.ParseOptions{ BaseURI: fileURIOf(path), // This document *was* retrieved by URI, so it has a dm:document-uri @@ -715,6 +706,21 @@ func (r *FileResolver) parseUncached( if r.ExternalEntities { opts.ExternalEntities = r } + return opts +} + +// parseUncached reads and parses a file. It holds no lock: this is the work +// that used to run under r.mu. +func (r *FileResolver) parseUncached( + path string, trackPos bool, b *xdm.EntityBudget) (*xdm.Tree, int64, error) { + // A file: URI rather than the bare path. This base URI is stamped on + // every element of the module and is what fn:document, fn:resolve-uri + // and fn:static-base-uri resolve against, and those are defined over + // URIs: an absolute filesystem path has no scheme, so it is still a + // *relative* URI reference and resolving against it drops everything + // before the last separator. resolvePath strips the scheme back off, so + // the filesystem sees the same path either way. + opts := r.parseOptions(path, trackPos) // The expansion is charged against the caller's shared allowance rather // than a fresh one per file. A nil budget leaves the per-document ceiling // in place. See ResolveModuleWith. diff --git a/xslt/runtime.go b/xslt/runtime.go index 48072ae4..0f7552f0 100644 --- a/xslt/runtime.go +++ b/xslt/runtime.go @@ -26,6 +26,10 @@ type runtime struct { ctx *xpath.Context + // bursts drives the loop of a streamable xsl:source-document running in + // bursts; see streamburst.go. + bursts *burstFeed + // absent records which groups of context components clearMergeContext, // withoutGroupingScope, clearRegexGroups and withOutputURI have already // bound to absent in ctx (or that nothing has bound yet), so that the clears every template and function call runs can diff --git a/xslt/sourcedoc.go b/xslt/sourcedoc.go index 8283e5d7..9d4ff56a 100644 --- a/xslt/sourcedoc.go +++ b/xslt/sourcedoc.go @@ -49,6 +49,9 @@ type sourceDocumentInstr struct { // expression -- a per-element property, not a per-module one. Taking the // module's base instead ignored xml:base, which non-stream-004 catches. baseURI string + // bursts is the plan for running the body without building the + // document, when it qualifies; see streamburst.go. + bursts *burstPlan } func (c *compiler) compileSourceDocument(n *xdm.Node) (Instruction, error) { @@ -66,12 +69,22 @@ func (c *compiler) compileSourceDocument(n *xdm.Node) (Instruction, error) { if err != nil { return nil, err } + var plan *burstPlan + if isYes(n.AttrValue("streamable")) { + plan = planBursts(n) + } + outer := c.burst + c.burst = plan body, err := c.compileSequence(n, n) + c.burst = outer if err != nil { return nil, err } + if plan != nil && plan.instr == nil { + plan = nil // the loop was not compiled (use-when) + } instr := &sourceDocumentInstr{href: href, validation: spec, body: body, - streamed: isYes(n.AttrValue("streamable")), baseURI: ns.baseURI} + streamed: isYes(n.AttrValue("streamable")), baseURI: ns.baseURI, bursts: plan} if n.Attr("", "use-accumulators") != nil { accums, err := parseUseAccumulators(n) if err != nil { @@ -87,6 +100,11 @@ func (i *sourceDocumentInstr) Execute(rt *runtime, out *outputBuilder) error { if err != nil { return err } + if i.bursts != nil { + if done, err := i.executeBursts(rt, out, href); done { + return err + } + } root, err := i.load(rt, href) if err != nil { return err @@ -130,19 +148,7 @@ func (i *sourceDocumentInstr) load(rt *runtime, href string) (*xdm.Node, error) "FODC0002: document access is disabled (no resolver configured): %q", href) } - // 18.1 resolves @href as fn:doc does, against the static base URI of the - // expression -- the base URI of the xsl:source-document element, which an - // xml:base on it or an ancestor may move. - base := i.baseURI - if base == "" { - base = rt.ctx.StaticBaseURI() - } - if base == "" { - if n, ok := rt.ctx.Item.(*xdm.Node); ok { - base = n.BaseURI() - } - } - tree, err := resolveDocumentIn(rt.ctx, href, base) + tree, err := resolveDocumentIn(rt.ctx, href, i.base(rt)) if err != nil { return nil, fmt.Errorf("FODC0002: cannot retrieve %q: %w", href, err) } @@ -163,6 +169,23 @@ func (i *sourceDocumentInstr) load(rt *runtime, href string) (*xdm.Node, error) return fragmentOf(rt.sheet.stripInputAnnotations(root), href) } +// base is the URI @href resolves against. 18.1 resolves @href as fn:doc +// does, against the static base URI of the expression -- the base URI of the +// xsl:source-document element, which an xml:base on it or an ancestor may +// move. +func (i *sourceDocumentInstr) base(rt *runtime) string { + base := i.baseURI + if base == "" { + base = rt.ctx.StaticBaseURI() + } + if base == "" { + if n, ok := rt.ctx.Item.(*xdm.Node); ok { + base = n.BaseURI() + } + } + return base +} + // fragmentOf applies the fragment identifier of href, if there is one, to the // document that href retrieved. // diff --git a/xslt/streamburst.go b/xslt/streamburst.go new file mode 100644 index 00000000..564ece04 --- /dev/null +++ b/xslt/streamburst.go @@ -0,0 +1,737 @@ +package xslt + +// Burst-mode execution of xsl:source-document streamable="yes". +// +// The narrowest common streaming pattern -- a streamable xsl:source-document +// whose body, under literal result elements at most, is one xsl:for-each or +// xsl:iterate over a child path of the document, reading only the item it is +// given -- is executed without building the document. xdm.ParseBursts hands +// over one selected element at a time, in a small tree of its own holding +// the element and copies of its ancestors, and the loop's body runs on it +// before the next one is read. +// +// That is correct only where the body cannot tell a burst from the whole +// tree, so the plan is made from the stylesheet source and fails closed: +// anything planBursts does not positively recognise leaves the instruction +// on the ordinary path, which reads the whole tree. The rules, and why each +// is needed, are in docs: DESIGN.md of the stream lane. In brief, a burst has +// no siblings, no following or preceding nodes, a root that holds only the +// one element, and ancestors that hold only it; and nodes of different +// bursts are different trees. So the body may not reach the root, may not +// use a sibling, following or preceding axis, may reach an ancestor only for +// its name or attributes, may not observe node identity (is, <<, >>, union, +// intersect, except, generate-id, fn:path), may not ask for last(), and may +// carry nothing but atomic values from one item to the next. + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/knroy/go-xml/v2/xdm" + "github.com/knroy/go-xml/v2/xpath" +) + +// noBursts turns burst execution off, and traceBursts, when set, is told of +// each run; both are for the tests that compare the two paths. +var ( + noBursts bool + traceBursts func(href string) +) + +// burstPlan is a streamable xsl:source-document that can run in bursts. +type burstPlan struct { + // path is the select expression's child steps, from the document node. + path []xdm.QName + // target is the xsl:for-each or xsl:iterate element, and instr its + // compiled form; sel is its select expression, evaluated against each + // burst's document node to apply the last step's predicates. + target *xdm.Node + instr Instruction + sel *xpath.Compiled +} + +// burstFeed is a burst plan being executed: the loop instruction it drives, +// and read, which calls each with every item the loop's select would have +// returned, in order. +type burstFeed struct { + instr Instruction + read func(each func(xdm.Item) error) error +} + +// burstResolver is a document resolver that can parse a document in bursts. +type burstResolver interface { + // wantsBursts reports whether uri should be parsed in bursts rather than + // resolved to a tree -- false when the resolver already holds the tree, + // which fn:doc must go on returning -- and the document URI the tree + // would have had. + wantsBursts(uri, base string) (docURI string, ok bool) + // parseBursts is xdm.ParseBursts over the document uri names, read with + // the options and limits ResolveDocument would read it with. An error + // from fn is returned as it is. + parseBursts(ctx *xpath.Context, uri, base string, path []xdm.QName, fn func(*xdm.Node) error) error +} + +// planBursts decides, from the stylesheet source, whether the streamable +// xsl:source-document sd can run in bursts. +func planBursts(sd *xdm.Node) *burstPlan { + for _, a := range []string{"validation", "type", "use-accumulators"} { + if sd.Attr("", a) != nil { + return nil + } + } + // The §19.8 verdict first: the body must be guaranteed-streamable, from + // constructs the analysis fully models. Everything below narrows that to + // what bursts can execute. + if p, known := analyzeSequenceConstructor(sd, postureStriding, nil, nil); !known || + !p.streamable() || p.posture != postureGrounded { + return nil + } + el := sd + var target *xdm.Node + for target == nil { + kid := onlyElementChild(el) + switch { + case kid == nil: + return nil + case isXSL(kid, "for-each") || isXSL(kid, "iterate"): + target = kid + case kid.Name().URI == xdm.NSXSL || !plainWrapper(kid): + return nil + default: + el = kid + } + } + for _, c := range target.ChildElements() { + if isXSL(c, "sort") { + return nil + } + } + path := burstPath(target) + if path == nil || !burstBody(target) { + return nil + } + return &burstPlan{path: path, target: target} +} + +// onlyElementChild is el's one element child, when el has exactly one and +// nothing else but white space. +func onlyElementChild(el *xdm.Node) *xdm.Node { + var only *xdm.Node + for ch := range el.Children() { + switch ch.Kind() { + case xdm.KindElement: + if only != nil { + return nil + } + only = ch + case xdm.KindText: + if strings.TrimSpace(ch.Value()) != "" { + return nil + } + case xdm.KindComment, xdm.KindPI: + default: + return nil + } + } + return only +} + +// plainWrapper reports whether a literal result element around the loop +// writes nothing but itself: no value template, which would be evaluated +// against the document node the bursts never build, and no xsl: attribute +// that changes what it constructs. +func plainWrapper(el *xdm.Node) bool { + for a := range el.Attrs() { + if !lreAttrOK(a) || strings.ContainsAny(a.Value(), "{}") { + return false + } + } + return true +} + +// lreAttrOK admits the xsl: attributes of a literal result element that do +// not change what it constructs. +func lreAttrOK(a *xdm.Node) bool { + if a.Name().URI != xdm.NSXSL { + return true + } + switch a.Name().Local { + case "exclude-result-prefixes", "extension-element-prefixes", "version", + "xpath-default-namespace", "expand-text", "default-collation", "use-when": + return true + } + return false +} + +// burstPath returns the element names of the loop's select, a path of child +// steps from the document node, or nil. Only the last step may have +// predicates, and they must not depend on position: a burst's element is +// the first and only one of its name in its tree. +func burstPath(target *xdm.Node) []xdm.QName { + e := parseAt(target, target.AttrValue("select")) + var steps []xpath.Expr + switch x := e.(type) { + case *xpath.PathExpr: + steps = x.Steps + case *xpath.Step: + steps = []xpath.Expr{x} + default: + return nil + } + var path []xdm.QName + for k, s := range steps { + st, ok := s.(*xpath.Step) + if !ok || st.Axis != xpath.AxisChild { + return nil + } + nt, ok := st.Test.(*xpath.NameTest) + if !ok || nt.AnyURI || nt.AnyLocal { + return nil + } + if len(st.Predicates) > 0 && k < len(steps)-1 { + return nil + } + for _, p := range st.Predicates { + if !booleanShaped(p) || !burstSafe(p, true) { + return nil + } + } + path = append(path, xdm.QName{URI: nt.Name.URI, Local: nt.Name.Local}) + } + return path +} + +// parseAt parses src as an expression written on el, or returns nil. +func parseAt(el *xdm.Node, src string) xpath.Expr { + if src == "" { + return nil + } + e, err := xpath.ParseVersion(src, newNSResolver(el, xpathDefaultNamespace(el)), xpathVersionAt(el)) + if err != nil { + return nil + } + return e +} + +// booleanShaped reports whether a predicate is certainly not numeric, so not +// positional: a comparison, a boolean connective or function, or a path. +func booleanShaped(e xpath.Expr) bool { + switch x := e.(type) { + case *xpath.BinaryOp: + switch x.Op { + case "=", "!=", "<", "<=", ">", ">=", "eq", "ne", "lt", "le", "gt", "ge", "and", "or": + return true + } + case *xpath.FuncCall: + if x.Name.URI == fnNS { + switch x.Name.Local { + case "not", "boolean", "exists", "empty", "true", "false", + "contains", "starts-with", "ends-with", "matches": + return true + } + } + case *xpath.PathExpr, *xpath.Step: + return true + } + return false +} + +// burstBody reports whether everything inside the loop element is something +// bursts execute exactly as the whole tree would. +func burstBody(target *xdm.Node) bool { + ok := true + walkElements(target, func(el *xdm.Node) bool { + if !ok { + return false + } + if !burstElement(el, target) { + ok = false + return false + } + for ch := range el.Children() { + // A text value template would need its own check; a text node + // with a brace is refused rather than told apart from one. + if ch.Kind() == xdm.KindText && strings.ContainsAny(ch.Value(), "{}") { + ok = false + return false + } + } + return true + }) + return ok +} + +// burstInstructions are the XSLT instructions a burst body may hold. Each +// one's expressions are checked on their own; what is missing either reads +// beyond the item (apply-templates and call-template reach code not checked +// here, xsl:number counts preceding nodes, xsl:key indexes the whole tree) +// or has effects the order of reading could show (xsl:message, +// xsl:result-document), or simply is not needed for the first stage. +var burstInstructions = map[string]bool{ + "for-each": true, "iterate": true, "if": true, "choose": true, "when": true, + "otherwise": true, "value-of": true, "copy-of": true, "sequence": true, + "text": true, "attribute": true, "element": true, "copy": true, + "variable": true, "comment": true, "processing-instruction": true, + "namespace": true, "param": true, "next-iteration": true, + "with-param": true, "break": true, "on-completion": true, "sort": true, + "map": true, "map-entry": true, +} + +func burstElement(el, target *xdm.Node) bool { + if el.Name().URI != xdm.NSXSL { + for a := range el.Attrs() { + if !lreAttrOK(a) || !avtSafe(el, a.Value()) { + return false + } + } + return true + } + if el != target && !burstInstructions[el.Name().Local] { + return false + } + for a := range el.Attrs() { + if a.Name().URI != "" { + continue + } + switch a.Name().Local { + case "select", "test": + if el == target { + continue // burstPath has checked it + } + e := parseAt(el, a.Value()) + if e == nil || !burstSafe(e, false) { + return false + } + case "use-attribute-sets", "validation", "type": + return false + default: + if !avtSafe(el, a.Value()) { + return false + } + } + } + if el.Parent() == target && isXSL(el, "param") && isXSL(target, "iterate") { + return iterateParamOK(el) + } + if isXSL(el, "with-param") && isXSL(el.Parent(), "next-iteration") { + return carriedAtomic(el) + } + return true +} + +// avtSafe checks the expressions of an attribute value template. +func avtSafe(el *xdm.Node, src string) bool { + exprs, ok := avtExpressions(src) + if !ok { + return false + } + for _, s := range exprs { + e := parseAt(el, s) + if e == nil || !burstSafe(e, false) { + return false + } + } + return true +} + +// iterateParamOK admits an xsl:param of the loop whose default does not read +// the focus. The default is evaluated with the xsl:iterate's own focus, +// which is the document node -- and bursts have no document node to give +// it. +func iterateParamOK(p *xdm.Node) bool { + if onlyElementChild(p) != nil || strings.TrimSpace(p.StringValue()) != "" { + return false + } + if s := p.AttrValue("select"); s != "" { + e := parseAt(p, s) + return e != nil && focusFree(e) + } + return true +} + +// carriedAtomic reports whether the value xsl:next-iteration carries to the +// next item can hold no node: a node of this burst carried into the next +// would be a node of a tree that no longer matches the document. The +// parameter's declared type decides when it is atomic; otherwise the +// expression must be one that only makes atomic values. +func carriedAtomic(wp *xdm.Node) bool { + sel := wp.AttrValue("select") + if sel == "" { + // A sequence constructor makes a new temporary tree: it is a copy. + return wp.Attr("", "select") == nil + } + if p := iterateParamNamed(wp); p != nil { + if st, err := xpath.ParseSequenceType(p.AttrValue("as"), newNSResolver(p, "")); err == nil && + p.AttrValue("as") != "" && (st.HasAtomicType || st.IsNumericType) { + return true + } + } + e := parseAt(wp, sel) + return e != nil && atomicValued(e) +} + +// iterateParamNamed finds the xsl:param of the innermost enclosing +// xsl:iterate that wp supplies. +func iterateParamNamed(wp *xdm.Node) *xdm.Node { + want, _ := paramName(wp) + it := enclosingIterate(wp) + if it == nil { + return nil + } + for _, c := range it.ChildElements() { + if isXSL(c, "param") { + if q, _ := paramName(c); q == want { + return c + } + } + } + return nil +} + +// atomicValued reports, conservatively, whether e can only return atomic +// values. +func atomicValued(e xpath.Expr) bool { + switch x := e.(type) { + case *xpath.Literal: + return true + case *xpath.BinaryOp: + switch x.Op { + case "is", "<<", ">>", "union", "intersect", "except": + return false + } + return true + case *xpath.UnaryOp: + return true + case *xpath.CastExpr: + return true + case *xpath.InstanceOfExpr: + return true + case *xpath.IfExpr: + return atomicValued(x.Then) && atomicValued(x.Else) + case *xpath.SequenceExpr: + for _, it := range x.Items { + if !atomicValued(it) { + return false + } + } + return true + case *xpath.FuncCall: + if x.Name.URI == xdm.NSXS { + return true + } + if x.Name.URI != fnNS { + return false + } + switch x.Name.Local { + case "count", "sum", "avg", "min", "max", "string", "number", "concat", + "string-join", "data", "boolean", "not", "exists", "empty", + "string-length", "normalize-space", "upper-case", "lower-case", + "substring", "substring-before", "substring-after", "translate", + "round", "floor", "ceiling", "abs", "name", "local-name", + "namespace-uri", "position", "true", "false", "contains", + "starts-with", "ends-with", "format-number", "distinct-values", + "string-to-codepoints", "codepoints-to-string": + return true + } + } + return false +} + +// burstFnBanned are the fn functions a burst would answer differently: they +// reach the root or the whole document, observe node identity or position +// among all the items, or read the document's DTD. +var burstFnBanned = map[string]bool{ + "root": true, "id": true, "idref": true, "element-with-id": true, + "key": true, "generate-id": true, "path": true, "last": true, + "unparsed-entity-uri": true, "unparsed-entity-public-id": true, + "accumulator-before": true, "accumulator-after": true, + "function-lookup": true, "transform": true, "load-xquery-module": true, + "apply": true, "current-group": true, "current-grouping-key": true, + "current-merge-group": true, "current-merge-key": true, +} + +// nameOnlyFns read nothing of a node but its name, its URIs, or whether it +// is there, so an ancestor -- which a burst copies without its other +// children -- answers them as the whole tree's would. +var nameOnlyFns = map[string]bool{ + "name": true, "local-name": true, "namespace-uri": true, "node-name": true, + "count": true, "exists": true, "empty": true, +} + +// burstSafe reports whether e reads nothing a burst lacks. noPos refuses +// fn:position as well, for a predicate of the loop's own select, where it +// would count within the burst. +func burstSafe(e xpath.Expr, noPos bool) bool { + switch x := e.(type) { + case *xpath.Literal, *xpath.VarRef, *xpath.ContextItem: + return true + case *xpath.Step: + return pathSafe([]xpath.Expr{x}, noPos, false) + case *xpath.PathExpr: + return !x.Root && pathSafe(x.Steps, noPos, false) + case *xpath.FilterExpr: + return burstSafe(x.Base, noPos) && allBurstSafe(x.Predicates, noPos) + case *xpath.BinaryOp: + switch x.Op { + case "is", "<<", ">>", "union", "intersect", "except": + return false + } + return burstSafe(x.Left, noPos) && burstSafe(x.Right, noPos) + case *xpath.UnaryOp: + return burstSafe(x.Operand, noPos) + case *xpath.FuncCall: + return callSafe(x, noPos) + case *xpath.IfExpr: + return allBurstSafe([]xpath.Expr{x.Cond, x.Then, x.Else}, noPos) + case *xpath.SequenceExpr: + return allBurstSafe(x.Items, noPos) + case *xpath.LetExpr: + return allBurstSafe(append(bindingExprs(x.Bindings), x.Return), noPos) + case *xpath.ForExpr: + return allBurstSafe(append(bindingExprs(x.Bindings), x.Return), noPos) + case *xpath.QuantifiedExpr: + return allBurstSafe(append(bindingExprs(x.Bindings), x.Test), noPos) + case *xpath.InstanceOfExpr: + return burstSafe(x.Operand, noPos) + case *xpath.CastExpr: + return burstSafe(x.Operand, noPos) + case *xpath.TreatExpr: + return burstSafe(x.Operand, noPos) + case *xpath.SimpleMap: + return burstSafe(x.Left, noPos) && burstSafe(x.Right, noPos) + case *xpath.MapConstructor: + return allBurstSafe(x.Keys, noPos) && allBurstSafe(x.Values, noPos) + case *xpath.ArrayConstructor: + return allBurstSafe(x.Members, noPos) + } + return false +} + +func allBurstSafe(es []xpath.Expr, noPos bool) bool { + for _, e := range es { + if !burstSafe(e, noPos) { + return false + } + } + return true +} + +func callSafe(x *xpath.FuncCall, noPos bool) bool { + switch x.Name.URI { + case fnNS: + if burstFnBanned[x.Name.Local] || noPos && x.Name.Local == "position" { + return false + } + if nameOnlyFns[x.Name.Local] && len(x.Args) == 1 { + if p, ok := x.Args[0].(*xpath.PathExpr); ok && !p.Root && pathSafe(p.Steps, noPos, true) { + return true + } + if s, ok := x.Args[0].(*xpath.Step); ok && pathSafe([]xpath.Expr{s}, noPos, true) { + return true + } + } + case xdm.NSXS, xdm.NSMath, xdm.NSMap, xdm.NSArray: + default: + return false + } + return allBurstSafe(x.Args, noPos) +} + +// pathSafe checks the steps of a relative path. A step up -- parent, +// ancestor -- reaches a copy that has its name and attributes but not its +// other children, so after one only further steps up are allowed, and then +// either an attribute step to end the path or, when upEnd says the path is +// the argument of a name-only function, nothing more. +func pathSafe(steps []xpath.Expr, noPos, upEnd bool) bool { + up := false + for k, s := range steps { + st, ok := s.(*xpath.Step) + if !ok { + if up || !burstSafe(s, noPos) { + return false + } + continue + } + switch st.Axis { + case xpath.AxisParent, xpath.AxisAncestor, xpath.AxisAncestorOrSelf: + if len(st.Predicates) > 0 { + return false + } + up = true + continue + case xpath.AxisAttribute: + if up && k != len(steps)-1 { + return false + } + case xpath.AxisChild, xpath.AxisDescendant, xpath.AxisDescendantOrSelf, + xpath.AxisSelf, xpath.AxisNamespace: + if up { + return false + } + default: + return false + } + if !allBurstSafe(st.Predicates, noPos) { + return false + } + } + if up { + last, _ := steps[len(steps)-1].(*xpath.Step) + return upEnd || last != nil && last.Axis == xpath.AxisAttribute + } + return true +} + +// noteBurstLoop records instr as the compiled loop of the burst plan being +// compiled, when n is its loop element. +func (c *compiler) noteBurstLoop(n *xdm.Node, instr Instruction, sel *xpath.Compiled) { + if c.burst != nil && c.burst.target == n { + c.burst.instr, c.burst.sel = instr, sel + } +} + +// executeBursts runs i in bursts when its plan and its resolver allow, and +// reports whether it did. +func (i *sourceDocumentInstr) executeBursts(rt *runtime, out *outputBuilder, href string) (bool, error) { + if noBursts || validSourceDocumentURI(href) != nil || strings.ContainsRune(href, '#') { + return false, nil + } + // The resolvers Transform wraps around the caller's are looked through + // here, doing for this read what each does for a tree: readDocResolver + // records it for XTDE1500 and answers from its own memory when the + // transformation has read the document already, and stripSpaceResolver + // strips whitespace, which is done per burst instead. + base := i.base(rt) + pkg := packageOf(rt.ctx) + var ( + rd *readDocResolver + strip *Stylesheet + br burstResolver + ) + for d := rt.ctx.Env().Docs; br == nil; { + switch v := d.(type) { + case *readDocResolver: + if _, seen := v.docs[docKey{href, base, pkg}]; seen { + return false, nil + } + rd, d = v, v.inner + case *stripSpaceResolver: + strip, d = v.sheet, v.inner + case burstResolver: + br = v + default: + return false, nil + } + } + docURI, ok := br.wantsBursts(href, base) + if !ok { + return false, nil + } + if rd != nil && docURI != "" { + rd.read[docURI] = true + if rd.written[docURI] { + return true, fmt.Errorf( + "XTDE1500: this transformation reads %q, which xsl:result-document "+ + "has already written", docURI) + } + } + if traceBursts != nil { + traceBursts(href) + } + // The body's own focus is the document node, which the bursts do not + // build. Nothing planBursts admits reads it, so an empty document node + // with the document's URIs stands in for it. + stand := xdm.NewTree() + sub := rt.withCurrent(stand.Root, 1, 1).clearCurrentRule() + plan := i.bursts + sub.bursts = &burstFeed{instr: plan.instr, read: func(each func(xdm.Item) error) error { + var bodyErr error + err := br.parseBursts(rt.ctx, href, base, plan.path, func(rec *xdm.Node) error { + root := rec.Tree().Root + if strip != nil { + root = strip.stripWhitespaceFor(pkg, root) + } + items, err := plan.sel.Eval(sub.withCurrent(root, 1, 1).ctx) + if err != nil { + bodyErr = err + return err + } + for _, it := range items { + if err := each(it); err != nil { + bodyErr = err + return err + } + } + return nil + }) + if err != nil && err != bodyErr { + return fmt.Errorf("FODC0002: cannot retrieve %q: %w", href, err) + } + return err + }} + return true, execSequence(i.body, sub, out) +} + +// burstsFor returns the feed that drives instr, or nil when instr evaluates +// its select as usual. +func (rt *runtime) burstsFor(instr Instruction) *burstFeed { + if f := rt.bursts; f != nil && f.instr == instr { + return f + } + return nil +} + +// wantsBursts implements burstResolver. +func (r *FileResolver) wantsBursts(uri, base string) (string, bool) { + path, err := r.resolvePath(uri, base) + if err != nil { + return "", true // parseBursts reports the error as ResolveDocument would + } + r.mu.Lock() + defer r.mu.Unlock() + _, cached := r.cache[path] + return fileURIOf(path), !cached +} + +// parseBursts implements burstResolver. It reads the file as parseUncached +// does -- the same confinement, options, entity allowance and byte limit, and +// the same errors -- but in bursts, and caches nothing: there is no tree. +func (r *FileResolver) parseBursts(ctx *xpath.Context, uri, base string, + steps []xdm.QName, fn func(*xdm.Node) error) error { + + path, err := r.resolvePath(uri, base) + if err != nil { + return err + } + opts := r.parseOptions(path, false).WithEntityBudget(ctx.EntityBudget()) + var fnErr error + err = r.openConfined(path, func(f *os.File) error { + max, lim := r.byteLimit() + tooBig := fmt.Errorf("%s exceeds the %d byte limit", path, max) + var src io.Reader = f + if max >= 0 { + if fi, err := f.Stat(); err == nil && fi.Size() > max { + return tooBig + } + src = io.LimitReader(f, lim) + } + cr := &countedReader{r: src} + err := xdm.ParseBursts(cr, opts, steps, func(n *xdm.Node) error { + fnErr = fn(n) + return fnErr + }) + if err != nil && err == fnErr { + return err + } + if max >= 0 && cr.n > max { + return tooBig + } + if err != nil { + return fmt.Errorf("parsing %s: %w", path, err) + } + return nil + }) + return err +} diff --git a/xslt/streamburst_test.go b/xslt/streamburst_test.go new file mode 100644 index 00000000..a1f718a0 --- /dev/null +++ b/xslt/streamburst_test.go @@ -0,0 +1,140 @@ +package xslt + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +const burstDoc = ` + + + Bob + a + b + + + text +` + +// TestBurstsMatchTree runs each stylesheet with bursts and without, and +// requires the same output -- and that bursts ran exactly when expected. +func TestBurstsMatchTree(t *testing.T) { + for _, tc := range []struct { + name string + body string + bursts bool + extra string + }{ + {"for-each copy-of", ``, true, ""}, + {"absolute path, predicate, position", ``, true, ""}, + {"ancestor names and attributes", ``, true, ""}, + {"iterate total", ``, true, ""}, + {"iterate break", ``, true, ""}, + {"strip-space", ``, true, ``}, + {"last()", ``, false, ""}, + {"sibling axis", ``, false, ""}, + {"positional predicate", ``, false, ""}, + {"ancestor content", ``, false, ""}, + {"identity", ``, false, ""}, + {"sort", ``, false, ""}, + {"node carried", ``, false, ""}, + {"sibling instruction", ``, false, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + src := `` + + tc.extra + `` + + tc.body + `` + tree, err := burstRun(t, src, "doc.xml", burstDoc, true) + got, ran := tree, burstRuns + want, err2 := burstRun(t, src, "doc.xml", burstDoc, false) + if (err == nil) != (err2 == nil) || got != want { + t.Fatalf("bursts differ from the tree\nbursts: %q %v\ntree: %q %v", got, err, want, err2) + } + if ran != tc.bursts { + t.Fatalf("ran in bursts: %v, want %v (output %q)", ran, tc.bursts, got) + } + }) + } +} + +var burstRuns bool + +// burstRun compiles src, writes doc to a temporary directory as name, and +// runs the template "main" with or without bursts. +func burstRun(t testing.TB, src, name, doc string, bursts bool) (string, error) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, name), []byte(doc), 0o600); err != nil { + t.Fatal(err) + } + return burstRunIn(t, src, dir, bursts) +} + +func burstRunIn(t testing.TB, src, dir string, bursts bool) (string, error) { + tree, err := xdm.ParseString(src, xdm.ParseOptions{BaseURI: fileURIOf(filepath.Join(dir, "s.xsl"))}) + if err != nil { + t.Fatal(err) + } + burstRuns = false + sheet, err := Compile(tree.Root, CompileOptions{}) + if err != nil { + // Not streamable is XTSE3430 whichever way it would have run. + return "", err + } + docs, err := NewFileResolver(dir) + if err != nil { + t.Fatal(err) + } + noBursts = !bursts + traceBursts = func(string) { burstRuns = true } + defer func() { noBursts, traceBursts = false, nil }() + res, err := sheet.Transform(context.Background(), nil, TransformOptions{ + InitialTemplate: "main", Documents: docs}) + if err != nil { + return "", err + } + return res.String(), nil +} + +// A document malformed after the records is an error with bursts too, and +// after xsl:break as well. +func TestBurstsMalformedTail(t *testing.T) { + src := ` + + + ` + _, err := burstRun(t, src, "doc.xml", "", true) + if !burstRuns || err == nil || !strings.Contains(err.Error(), "FODC0002") { + t.Fatalf("bursts ran %v, error %v", burstRuns, err) + } + _, terr := burstRun(t, src, "doc.xml", "", false) + if terr == nil || !strings.Contains(terr.Error(), "FODC0002") { + t.Fatalf("tree error %v", terr) + } +} + +// Many records, totalled by xsl:iterate: the answer is the tree's, and it +// came from bursts. (Memory is measured by the lane's harness, not here.) +func TestBurstsManyRecords(t *testing.T) { + var b strings.Builder + b.WriteString("") + for i := range 20000 { + fmt.Fprintf(&b, "record %d\n", i%10, i) + } + b.WriteString("") + src := ` + + + + + ` + out, err := burstRun(t, src, "big.xml", b.String(), true) + if err != nil || !strings.HasSuffix(out, "90000") || !burstRuns { + t.Fatalf("got %q, %v, bursts %v", out, err, burstRuns) + } +} From 09f0c5dbec30ab0450ebbae9842d6337039c2e25 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 13:57:25 -0500 Subject: [PATCH 08/45] xdm: keep the text store its size; ws table behind aux pointer --- xdm/record.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/xdm/record.go b/xdm/record.go index f0b0e285..4c5fec43 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1678,12 +1678,18 @@ type textStore struct { blocks [][]byte cur int32 // the ordinary block values are appended to, + 1 (0: none) size int32 // capacity of the last ordinary block made + // aux holds what few trees use, behind one pointer so that the store + // stays the size it was. + aux *textAux +} + +type textAux struct { // shared maps a string registered by share to where it is stored, so that // add hands the same range to every value that is that string. shared map[sharedKey][2]uint32 // ws shares one stored value among the whitespace-only values add is - // given, as a parse does; made on the first. - ws *spaceTable + // given, as a parse does. + ws spaceTable } // sharedKey identifies a string by where it lives: the same storage is the @@ -1721,14 +1727,14 @@ func (s *textStore) add(v string) (uint32, uint32) { } if len(v) <= maxSpaceLen { if b := unsafe.Slice(unsafe.StringData(v), len(v)); onlySpace(b) { - if s.ws == nil { - s.ws = new(spaceTable) + if s.aux == nil { + s.aux = new(textAux) } - return s.ws.text(s, b) + return s.aux.ws.text(s, b) } } - if s.shared != nil && len(v) >= shareMin { - if r, ok := s.shared[sharedKey{unsafe.StringData(v), len(v)}]; ok { + if s.aux != nil && len(v) >= shareMin { + if r, ok := s.aux.shared[sharedKey{unsafe.StringData(v), len(v)}]; ok { return r[0], r[1] } } @@ -1750,13 +1756,16 @@ func (s *textStore) share(v string) { if len(v) < shareMin { return } - if s.shared == nil { - s.shared = map[sharedKey][2]uint32{} + if s.aux == nil { + s.aux = new(textAux) + } + if s.aux.shared == nil { + s.aux.shared = map[sharedKey][2]uint32{} } k := sharedKey{unsafe.StringData(v), len(v)} - if _, ok := s.shared[k]; !ok { + if _, ok := s.aux.shared[k]; !ok { off, n := s.add(v) - s.shared[k] = [2]uint32{off, n} + s.aux.shared[k] = [2]uint32{off, n} } } From 80f999f396d21f1c5d863cd5b5813de4abf5448e Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 14:01:53 -0500 Subject: [PATCH 09/45] xdm, xslt: bursts are bulk clones of the spine; FileResolver.MaxNodes; unfiltered loops skip select --- xdm/bursts.go | 76 ++++++++++++++------------------------------- xdm/parse.go | 4 ++- xslt/resolver.go | 10 ++++++ xslt/streamburst.go | 32 ++++++++++++------- 4 files changed, 57 insertions(+), 65 deletions(-) diff --git a/xdm/bursts.go b/xdm/bursts.go index 42ceca84..431ec095 100644 --- a/xdm/bursts.go +++ b/xdm/bursts.go @@ -1,6 +1,10 @@ package xdm -import "io" +import ( + "io" + + "github.com/knroy/go-xml/v2/internal/xdmclone" +) // Burst-mode parsing: the incremental half of streamed XSLT execution. // @@ -109,72 +113,38 @@ func (b *burster) discardLast(p *Node, spaces *spaceTable) { p.RemoveLastChild() p.tree.text.rewind(b.marks[len(b.marks)-1]) // The whitespace table remembers ranges in the store, and some of them - // were just rewound. - *spaces = spaceTable{} + // were just released. Its map is not rebuilt for the next burst. + *spaces = spaceTable{noMap: true} } -// deliver copies el and its ancestry into a tree of its own and hands it to -// fn. +// deliver hands el to fn in a tree of its own. By now the parse's tree holds +// nothing but the document node, any prolog comments and processing +// instructions, the spine and el, which is exactly the burst's tree: it is +// copied in bulk, sharing the names, frames and text the records point into. func (b *burster) deliver(el *Node) error { - src := el.tree - t := NewTree() - t.XMLVersion = "1.0" + m := cloneSubtree(el.tree.Root, xdmclone.Options{}) + c := m(el) + c.tree.XMLVersion = "1.0" if b.v11 { - t.XMLVersion = "1.1" - } - t.Root.SetBaseURI(src.Root.BaseURI()) - t.Root.SetDocumentURI(src.Root.DocumentURI()) - var chain []*Node - for a := el.Parent(); a != nil && a.Kind() == KindElement; a = a.Parent() { - chain = append(chain, a) - } - p := t.Root - for i := len(chain) - 1; i >= 0; i-- { - a := chain[i] - c := p.AppendShallowCopy(a) - for prefix, uri := range a.DeclaredNamespaces() { - c.AddNamespace(prefix, uri) - } - for at := range a.Attrs() { - c.AppendAttr(at.Name(), at.Value()).CopyTypingFrom(at) - } - p = c + c.tree.XMLVersion = "1.1" } - c := p.AppendCopy(el) - t.Finalize() return b.fn(c) } // textMark is a point in a textStore that rewind returns it to. -type textMark struct { - blocks int - cur int32 - size int32 - curLen int - hasBlock bool -} +type textMark struct{ blocks int } -func (s *textStore) mark() textMark { - m := textMark{blocks: len(s.blocks), cur: s.cur, size: s.size} - if s.cur > 0 { - m.curLen = len(s.blocks[s.cur-1]) - m.hasBlock = true - } - return m -} +func (s *textStore) mark() textMark { return textMark{len(s.blocks)} } -// rewind forgets every value stored since m. Nothing may still read one of -// them: ParseBursts only rewinds over records it has removed, and the trees it -// hands out copy their values (a value long enough to be aliased rather than -// copied lives in a block of its own, which rewind releases and never -// rewrites). +// rewind forgets every value stored since m. A burst's tree shares the +// blocks those values are in, so they are released rather than reused: the +// block that was current at m is closed to further values, and the blocks +// made since are dropped from the store -- the burst trees that read them +// keep them alive as long as they need them. func (s *textStore) rewind(m textMark) { for i := m.blocks; i < len(s.blocks); i++ { s.blocks[i] = nil } s.blocks = s.blocks[:m.blocks] - s.cur, s.size = m.cur, m.size - if m.hasBlock { - s.blocks[s.cur-1] = s.blocks[s.cur-1][:m.curLen] - } + s.cur, s.size = 0, 0 } diff --git a/xdm/parse.go b/xdm/parse.go index 7b276ac5..48c7c136 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -1017,6 +1017,8 @@ func (r *textRun) flush(spaces *spaceTable, strip func(*Node) bool) { type spaceTable struct { m map[string][2]uint32 byLen [maxSpaceLen + 1][2]uint32 + // noMap keeps m unmade: ParseBursts empties the table at every burst. + noMap bool } const ( @@ -1038,7 +1040,7 @@ func (t *spaceTable) text(s *textStore, b []byte) (uint32, uint32) { off, n := s.addBytes(b, 0) r := [2]uint32{off, n} t.byLen[len(b)] = r - if len(t.m) < maxSpaceTable { + if len(t.m) < maxSpaceTable && !t.noMap { if t.m == nil { t.m = map[string][2]uint32{} } diff --git a/xslt/resolver.go b/xslt/resolver.go index f5a9889f..edd660c5 100644 --- a/xslt/resolver.go +++ b/xslt/resolver.go @@ -86,6 +86,14 @@ type FileResolver struct { // primitive. MaxBytes int64 + // MaxNodes bounds the nodes of one document this resolver parses, as + // xdm.ParseOptions.MaxNodes does: zero means xdm.DefaultMaxNodes, and a + // negative value no limit. Like MaxBytes, which is also the parser's byte + // limit, it is counted over the whole document even when a streamable + // xsl:source-document reads it without building it -- so raising both is + // what lets such a stylesheet read a document of any size. + MaxNodes int + // There is deliberately no network option. Adding one would mean this // type could not be recommended without caveats, and a validator has no // need to fetch rule sets at transform time. @@ -702,6 +710,8 @@ func (r *FileResolver) parseOptions(path string, trackPos bool) xdm.ParseOptions DocumentURI: fileURIOf(path), AllowDOCTYPE: r.AllowDOCTYPE, TrackPositions: trackPos, + MaxBytes: r.MaxBytes, + MaxNodes: r.MaxNodes, } if r.ExternalEntities { opts.ExternalEntities = r diff --git a/xslt/streamburst.go b/xslt/streamburst.go index 564ece04..cd7dc6ca 100644 --- a/xslt/streamburst.go +++ b/xslt/streamburst.go @@ -50,6 +50,9 @@ type burstPlan struct { target *xdm.Node instr Instruction sel *xpath.Compiled + // filtered says the last step has predicates. Without them each burst's + // element is the item, and sel need not be evaluated. + filtered bool } // burstFeed is a burst plan being executed: the loop instruction it drives, @@ -108,11 +111,11 @@ func planBursts(sd *xdm.Node) *burstPlan { return nil } } - path := burstPath(target) + path, filtered := burstPath(target) if path == nil || !burstBody(target) { return nil } - return &burstPlan{path: path, target: target} + return &burstPlan{path: path, target: target, filtered: filtered} } // onlyElementChild is el's one element child, when el has exactly one and @@ -169,7 +172,7 @@ func lreAttrOK(a *xdm.Node) bool { // steps from the document node, or nil. Only the last step may have // predicates, and they must not depend on position: a burst's element is // the first and only one of its name in its tree. -func burstPath(target *xdm.Node) []xdm.QName { +func burstPath(target *xdm.Node) (path []xdm.QName, filtered bool) { e := parseAt(target, target.AttrValue("select")) var steps []xpath.Expr switch x := e.(type) { @@ -178,29 +181,29 @@ func burstPath(target *xdm.Node) []xdm.QName { case *xpath.Step: steps = []xpath.Expr{x} default: - return nil + return nil, false } - var path []xdm.QName for k, s := range steps { st, ok := s.(*xpath.Step) if !ok || st.Axis != xpath.AxisChild { - return nil + return nil, false } nt, ok := st.Test.(*xpath.NameTest) if !ok || nt.AnyURI || nt.AnyLocal { - return nil + return nil, false } if len(st.Predicates) > 0 && k < len(steps)-1 { - return nil + return nil, false } for _, p := range st.Predicates { if !booleanShaped(p) || !burstSafe(p, true) { - return nil + return nil, false } } + filtered = len(st.Predicates) > 0 path = append(path, xdm.QName{URI: nt.Name.URI, Local: nt.Name.Local}) } - return path + return path, filtered } // parseAt parses src as an expression written on el, or returns nil. @@ -651,7 +654,14 @@ func (i *sourceDocumentInstr) executeBursts(rt *runtime, out *outputBuilder, hre err := br.parseBursts(rt.ctx, href, base, plan.path, func(rec *xdm.Node) error { root := rec.Tree().Root if strip != nil { - root = strip.stripWhitespaceFor(pkg, root) + root, rec = strip.stripWhitespaceForFrom(pkg, root, rec) + } + if !plan.filtered { + if err := each(rec); err != nil { + bodyErr = err + return err + } + return nil } items, err := plan.sel.Eval(sub.withCurrent(root, 1, 1).ctx) if err != nil { From 546af03e4ba827ebfcf9a477c7d0e1d5ef9a2920 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 14:43:10 -0500 Subject: [PATCH 10/45] xdm: a deep copy borrows source text blocks it copies most of --- xdm/build.go | 39 +++++++++++++--- xdm/bursts.go | 3 ++ xdm/record.go | 102 ++++++++++++++++++++++++++++++++++++++++++ xdm/textshare_test.go | 79 ++++++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 xdm/textshare_test.go diff --git a/xdm/build.go b/xdm/build.go index 84d19f63..7d725d46 100644 --- a/xdm/build.go +++ b/xdm/build.go @@ -52,19 +52,43 @@ func (p *Node) AppendPI(target, value string) *Node { } func (p *Node) appendLeaf(kind NodeKind, name QName, value string) *Node { + return p.appendLeafFrom(kind, name, value, nil) +} + +// appendLeafFrom is appendLeaf for a copy of from, a leaf of another tree +// whose value is value: the copy's text may share the original's blocks. +func (p *Node) appendLeafFrom(kind NodeKind, name QName, value string, from *Node) *Node { c := p.appendChild(kind) c.name = p.tree.intern(name) - if value != "" { + if from = lender(p, from); from != nil { + c.v0, c.v1 = p.tree.text.addFrom(&from.tree.text, from.v0, from.v1) + } else if value != "" { c.v0, c.v1 = p.tree.text.add(value) } return c } +// lender returns from if the text of a copy of it appended to p can be taken +// from its store: from is a leaf of another tree that keeps its value in its +// text store. +func lender(p, from *Node) *Node { + if from == nil || from.tree == nil || from.tree == p.tree || from.flags&fSide != 0 || !from.isLeaf() { + return nil + } + return from +} + // AppendAttr appends an attribute to p, which must be an element with no // children yet, and returns it. (A document node may carry one too, for a // caller assembling a tree out of a result sequence that holds a parentless // attribute; nothing the data model builds does that.) func (p *Node) AppendAttr(name QName, value string) *Node { + return p.appendAttrFrom(name, value, nil) +} + +// appendAttrFrom is AppendAttr for a copy of from, an attribute of another +// tree whose value is value. +func (p *Node) appendAttrFrom(name QName, value string, from *Node) *Node { t := p.tree t.openTo(p) if p.isLeaf() || p.v1 != noIdx || t.n != p.self+1+p.attrCount() { @@ -75,7 +99,9 @@ func (p *Node) AppendAttr(name QName, value string) *Node { a.parent = p.self a.v1 = 0 a.name = t.intern(name) - if value != "" { + if from = lender(p, from); from != nil { + a.v0, a.v1 = t.text.addFrom(&from.tree.text, from.v0, from.v1) + } else if value != "" { a.v0, a.v1 = t.text.add(value) } if k := p.attrCount() + 1; k < 0xFFFF && p.flags&fManyAttrs == 0 { @@ -97,6 +123,9 @@ func (p *Node) AppendCopy(src *Node) *Node { // A subtree still being built in p's own tree grows as the copy is // appended; the copy stops where src ended when it began. limit := src.tree.n + if !src.isLeaf() && src.flags&fSide == 0 { + p.tree.text.planFrom(src.tree, src.self, min(src.tree.endOf(src), limit)) + } c := p.AppendShallowCopy(src) fillCopy(c, src, nil, limit) return c @@ -109,12 +138,12 @@ func (p *Node) AppendShallowCopy(src *Node) *Node { var c *Node switch src.Kind() { case KindAttribute: - c = p.AppendAttr(src.Name(), src.Value()) + c = p.appendAttrFrom(src.Name(), src.Value(), src) case KindElement, KindDocument: c = p.appendChild(src.Kind()) c.name = p.tree.intern(src.Name()) default: - c = p.appendLeaf(src.Kind(), src.Name(), src.Value()) + c = p.appendLeafFrom(src.Kind(), src.Name(), src.Value(), src) } c.SetBaseURI(src.BaseURI()) c.CopyTypingFrom(src) @@ -157,7 +186,7 @@ func fillCopy(c, src *Node, drop func(*Node) bool, limit uint32) { c.AddNamespace(prefix, uri) } for a := range src.Attrs() { - c.AppendAttr(a.Name(), a.Value()).CopyTypingFrom(a) + c.appendAttrFrom(a.Name(), a.Value(), a).CopyTypingFrom(a) } if src.v1 == noIdx { return diff --git a/xdm/bursts.go b/xdm/bursts.go index 431ec095..5737eb64 100644 --- a/xdm/bursts.go +++ b/xdm/bursts.go @@ -147,4 +147,7 @@ func (s *textStore) rewind(m textMark) { } s.blocks = s.blocks[:m.blocks] s.cur, s.size = 0, 0 + if s.aux != nil { + s.aux.lent, s.aux.last = nil, nil + } } diff --git a/xdm/record.go b/xdm/record.go index 94ce6b92..506fbc0d 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1685,6 +1685,18 @@ type textAux struct { // ws shares one stored value among the whitespace-only values add is // given, as a parse does. ws spaceTable + // lent counts, per block of another tree's store (keyed by address, which + // is only a count while the block is not borrowed), the bytes addFrom was + // asked to copy from it, and records where a borrowed block sits here. + lent map[uintptr]*lendState + // lastKey/last cache the previous lookup; copies run in document order. + lastKey uintptr + last *lendState +} + +type lendState struct { + used int32 // bytes asked for from the block so far + idx int32 // index in this store of the borrowed block, + 1 (0: not borrowed) } // sharedKey identifies a string by where it lives: the same storage is the @@ -1764,6 +1776,96 @@ func (s *textStore) share(v string) { } } +// addFrom stores the value at (off, n) of src, the store of another tree, as +// add would store its string, but lets the range name src's block instead of +// a copy of the bytes once this store has been asked for half of the block's +// bytes: from then on every value of that block is free. A borrowed block is +// the source's slice with its capacity clipped to its length, so nothing is +// ever appended into it from here, and the source only appends past that +// length; the bytes below it never change. Borrowing pins the whole block, at +// worst twice the bytes this store asked of it, which bounds what a small +// result can retain of a large source. +func (s *textStore) addFrom(src *textStore, off, n uint32) (uint32, uint32) { + if n == 0 { + return 0, 0 + } + k := off >> textShift + blk := src.blocks[k] + // Whitespace goes through add's table; a value of a block of its own is + // kept by reference there already; and the store's block numbers must not + // run out for want of room to copy into. + if s == src || n >= textOwnFrom || len(s.blocks) >= 1<<(31-textShift) || + n <= maxSpaceLen && onlySpace(blk[off&textMask:][:n]) { + return s.add(src.str(off, n)) + } + l := s.lend(blk) + if l.idx == 0 { + l.used += int32(n) + if 2*int(l.used) < len(blk) { + return s.add(src.str(off, n)) + } + s.newBlock(blk[:len(blk):len(blk)]) + l.idx = int32(len(s.blocks)) + } + if int(off&textMask)+int(n) > len(s.blocks[l.idx-1]) { + return s.add(src.str(off, n)) // stored after the block was borrowed + } + return uint32(l.idx-1)<>textShift][r.v0&textMask:][:r.v1] + if len(b) > maxSpaceLen || !onlySpace(b) { + counts[r.v0>>textShift] += int32(len(b)) + } + } + for k, c := range counts { + if blk := t.text.blocks[k]; c > 0 && 2*int(c) >= len(blk) && len(blk) > 0 { + s.lend(blk).used = c + } + } +} + +// planMin is the fewest records planFrom walks. +const planMin = 4096 + +// lend returns the account of the block blk of another store. +func (s *textStore) lend(blk []byte) *lendState { + a := s.aux + if a == nil { + a = new(textAux) + s.aux = a + } + key := uintptr(unsafe.Pointer(unsafe.SliceData(blk))) + if l := a.last; l != nil && a.lastKey == key { + return l + } + l := a.lent[key] + if l == nil { + if a.lent == nil { + a.lent = map[uintptr]*lendState{} + } + l = new(lendState) + a.lent[key] = l + } + a.lastKey, a.last = key, l + return l +} + // keep stores b, which nothing else writes to, as a block of its own. func (s *textStore) keep(b []byte) (uint32, uint32) { s.newBlock(b) diff --git a/xdm/textshare_test.go b/xdm/textshare_test.go new file mode 100644 index 00000000..058d7d5c --- /dev/null +++ b/xdm/textshare_test.go @@ -0,0 +1,79 @@ +package xdm + +import ( + "fmt" + "strings" + "testing" + "unsafe" +) + +// sharesBlock reports whether dst's store holds a block that is src's. +func sharesBlock(dst, src *Tree) int { + n := 0 + for _, d := range dst.text.blocks { + for _, s := range src.text.blocks { + if len(d) > 0 && len(s) > 0 && unsafe.SliceData(d) == unsafe.SliceData(s) { + n++ + } + } + } + return n +} + +// textShareSource builds a tree of n children, each holding a distinct text +// of about 40 bytes, in a store that is still being appended to. +func textShareSource(n int) (*Tree, *Node) { + src := NewTree() + root := src.Root.AppendElement(QName{Local: "r"}) + for i := range n { + c := root.AppendElement(QName{Local: "e"}) + c.AppendAttr(QName{Local: "a"}, fmt.Sprintf("attribute value number %08d padded", i)) + c.AppendText(fmt.Sprintf("element text number %08d, a bit longer", i)) + } + return src, root +} + +// A copy of a large part of a tree borrows its text blocks; a copy of a +// sliver of it does not pin them. Whatever it borrows, the copy's values stay +// what they were when the source or the copy is added to afterwards. +func TestCopyTextSharesBlocksOnlyWhenItCopiesMost(t *testing.T) { + src, root := textShareSource(6000) + + dst := NewTree() + cp := dst.Root.AppendCopy(root) + if got := sharesBlock(dst, src); got == 0 { + t.Fatal("a copy of the whole tree borrowed none of its text blocks") + } + + small := NewTree() + i := 0 + for c := range root.Children() { + if i++; i == 4000 { + small.Root.AppendCopy(c) + break + } + } + if got := sharesBlock(small, src); got != 0 { + t.Fatalf("a copy of one child pinned %d blocks of the source", got) + } + + // The last text of the copy is the last value of the source's open block. + // Appending to it must land in storage of the copy's own, and the source + // appending after the borrowed length must not reach the copy. + last := cp.LastChild().LastChild() + srcLast := root.LastChild().LastChild() + want := last.Value() + last.AppendValue("XYZ") + root.AppendElement(QName{Local: "x"}).AppendText("QQQQ") + if got := last.Value(); got != want+"XYZ" { + t.Fatalf("copied text changed under the copy: %q, want %q", got, want+"XYZ") + } + if got := srcLast.Value(); !strings.HasPrefix(got, "element text number 00005999") || got != want { + t.Fatalf("source text changed: %q, want %q", got, want) + } + for i, c := 0, cp.FirstChild(); c != nil && i < 6000; i, c = i+1, c.NextSibling() { + if v := c.Attr("", "a").Value(); v != fmt.Sprintf("attribute value number %08d padded", i) { + t.Fatalf("attribute %d: %q", i, v) + } + } +} From 27757b3e54b1274ffcd3b123c381843c6badc956 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 14:51:20 -0500 Subject: [PATCH 11/45] xdm, xslt: fix three burst-mode differences from tree mode Rewinding the text store between bursts left the shared whitespace table pointing into released blocks: whitespace-only attribute values read other bytes, or the parse panicked. A path predicate whose last step is not an axis step can be numeric, so positional; the gate no longer admits it. A document that fails to parse in bursts no longer counts as read for XTDE1500, as in tree mode. --- xdm/bursts.go | 5 +++ xslt/streamburst.go | 17 ++++++++- xslt/streamburst_test.go | 78 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) diff --git a/xdm/bursts.go b/xdm/bursts.go index 431ec095..52747758 100644 --- a/xdm/bursts.go +++ b/xdm/bursts.go @@ -147,4 +147,9 @@ func (s *textStore) rewind(m textMark) { } s.blocks = s.blocks[:m.blocks] s.cur, s.size = 0, 0 + // The shared whitespace values may lie in the blocks just dropped. The + // values share registers come from the DTD, stored before any mark. + if s.aux != nil { + s.aux.ws = spaceTable{} + } } diff --git a/xslt/streamburst.go b/xslt/streamburst.go index cd7dc6ca..6f3a4abc 100644 --- a/xslt/streamburst.go +++ b/xslt/streamburst.go @@ -235,8 +235,16 @@ func booleanShaped(e xpath.Expr) bool { return true } } - case *xpath.PathExpr, *xpath.Step: + case *xpath.Step: return true + case *xpath.PathExpr: + // A path's last step can be any expression, a number among them + // (@k/xs:integer(.)), which would make the predicate positional. + if len(x.Steps) == 0 { + return true // "/" + } + _, ok := x.Steps[len(x.Steps)-1].(*xpath.Step) + return ok } return false } @@ -632,7 +640,13 @@ func (i *sourceDocumentInstr) executeBursts(rt *runtime, out *outputBuilder, hre if !ok { return false, nil } + // Tree mode records the document only once it has loaded; a document + // that fails to parse must not count as read here either (unread). + unread := func() {} if rd != nil && docURI != "" { + if !rd.read[docURI] { + unread = func() { delete(rd.read, docURI) } + } rd.read[docURI] = true if rd.written[docURI] { return true, fmt.Errorf( @@ -677,6 +691,7 @@ func (i *sourceDocumentInstr) executeBursts(rt *runtime, out *outputBuilder, hre return nil }) if err != nil && err != bodyErr { + unread() return fmt.Errorf("FODC0002: cannot retrieve %q: %w", href, err) } return err diff --git a/xslt/streamburst_test.go b/xslt/streamburst_test.go index a1f718a0..5a71ebf3 100644 --- a/xslt/streamburst_test.go +++ b/xslt/streamburst_test.go @@ -138,3 +138,81 @@ func TestBurstsManyRecords(t *testing.T) { t.Fatalf("got %q, %v, bursts %v", out, err, burstRuns) } } + +// Inputs on which bursts once differed from the tree: whitespace-only +// attribute values read from released text blocks (a wrong value, or a +// panic), and a path predicate ending in a number, which is positional. +func TestBurstsMatchTreeOn(t *testing.T) { + for _, tc := range []struct { + name, body, doc string + bursts bool + }{ + {"space attributes", ``, + ``, true}, + {"space attributes, three records", ``, + ``, true}, + {"space attributes on the spine", ``, + ``, true}, + {"numeric path predicate", ``, + ``, false}, + {"axis path predicate", ``, + ``, true}, + } { + t.Run(tc.name, func(t *testing.T) { + src := `` + + `` + + tc.body + `` + got, err := burstRun(t, src, "doc.xml", tc.doc, true) + ran := burstRuns + want, err2 := burstRun(t, src, "doc.xml", tc.doc, false) + if (err == nil) != (err2 == nil) || got != want { + t.Fatalf("bursts differ from the tree\nbursts: %q %v\ntree: %q %v", got, err, want, err2) + } + if ran != tc.bursts { + t.Fatalf("ran in bursts: %v, want %v (output %q)", ran, tc.bursts, got) + } + }) + } +} + +// A document that fails to parse is not read, in bursts as in the tree: once +// xsl:try has caught the FODC0002, writing that URI is not XTDE1500. +func TestBurstsFailedReadIsNotRead(t *testing.T) { + src := ` + + + + ` + for _, bursts := range []bool{true, false} { + // The resolver records the path with symlinks resolved (macOS + // temporary directories are under one), so the output must be too. + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "doc.xml"), []byte(""), 0o600); err != nil { + t.Fatal(err) + } + tree, err := xdm.ParseString(src, xdm.ParseOptions{BaseURI: fileURIOf(filepath.Join(dir, "s.xsl"))}) + if err != nil { + t.Fatal(err) + } + sheet, err := Compile(tree.Root, CompileOptions{}) + if err != nil { + t.Fatal(err) + } + docs, err := NewFileResolver(dir) + if err != nil { + t.Fatal(err) + } + burstRuns = false + noBursts = !bursts + traceBursts = func(string) { burstRuns = true } + _, err = sheet.Transform(context.Background(), nil, TransformOptions{InitialTemplate: "main", + Documents: docs, BaseOutputURI: fileURIOf(filepath.Join(dir, "out.xml"))}) + noBursts, traceBursts = false, nil + if err != nil || burstRuns != bursts { + t.Errorf("bursts %v (ran %v): %v", bursts, burstRuns, err) + } + } +} From 907c6742791a373455f8be39f2b86318c25caaaa Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 14:52:41 -0500 Subject: [PATCH 12/45] xdm: allocate full record chunks in pairs so the malloc header pads nothing --- xdm/clone.go | 6 +++--- xdm/parse_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ xdm/record.go | 19 ++++++++++++++++++- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/xdm/clone.go b/xdm/clone.go index c90e2016..67f6a0aa 100644 --- a/xdm/clone.go +++ b/xdm/clone.go @@ -129,9 +129,9 @@ func cloneSubtree(top *Node, o xdmclone.Options) func(*Node) *Node { total = newIdx[n] } last, _ := chunkOf(total - 1) - c.chunks = make([][]Node, last+1) - for k := range c.chunks { - c.chunks[k] = make([]Node, chunkLen(k)) + c.chunks = make([][]Node, 0, last+1) + for len(c.chunks) <= last { + c.chunks = appendChunk(c.chunks, len(c.chunks), last) } c.n = total diff --git a/xdm/parse_test.go b/xdm/parse_test.go index 3ed8d688..3bd74ab2 100644 --- a/xdm/parse_test.go +++ b/xdm/parse_test.go @@ -5,6 +5,8 @@ import ( "strings" "testing" "unsafe" + + "github.com/knroy/go-xml/v2/internal/xdmclone" ) func TestParseBasicTree(t *testing.T) { @@ -572,3 +574,43 @@ func TestTextRunPieces(t *testing.T) { } } } + +// TestFullChunksPaired pins the footprint of full record chunks. One full +// chunk is 20480 B; with the runtime's 8-byte malloc header it falls in the +// 21760 B size class and wastes 5%. From pairFrom on, chunks are allocated in +// pairs (40960 B, whole pages, no header), so the two halves of a pair must be +// adjacent in memory and each capped at chunkMax. The same holds for a copy. +func TestFullChunksPaired(t *testing.T) { + if unsafe.Sizeof(Node{}) != 40 { + t.Skip("pinned for the 40-byte record") + } + if 2*chunkMax*40%8192 != 0 { + t.Fatalf("a chunk pair is %d B, not whole 8 KiB pages", 2*chunkMax*40) + } + n := (growingTotal + (pairFrom+4)*chunkMax) // elements: past pairFrom + 2 pairs + doc := "" + strings.Repeat("", n) + "" + tree, err := ParseString(doc, ParseOptions{}) + if err != nil { + t.Fatal(err) + } + check := func(name string, tr *Tree) { + t.Helper() + pairs := 0 + for k := chunkMaxLog + 1 + pairFrom; k+1 < len(tr.chunks); k += 2 { + a, b := tr.chunks[k], tr.chunks[k+1] + if cap(a) != chunkMax || len(b) != chunkMax { + t.Fatalf("%s: chunk %d len/cap %d/%d, want %d", name, k, len(a), cap(a), chunkMax) + } + if uintptr(unsafe.Pointer(&b[0]))-uintptr(unsafe.Pointer(&a[0])) != chunkMax*40 { + t.Fatalf("%s: chunks %d and %d are not one allocation", name, k, k+1) + } + pairs++ + } + if pairs < 2 { + t.Fatalf("%s: %d chunk pairs, want at least 2", name, pairs) + } + } + check("parse", tree) + cp := cloneSubtree(tree.Root, xdmclone.Options{})(tree.Root) + check("clone", cp.tree) +} diff --git a/xdm/record.go b/xdm/record.go index 94ce6b92..9d5aafd8 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -333,6 +333,23 @@ func chunkOf(i uint32) (k int, off uint32) { return chunkMaxLog + 1 + int(j/chunkMax), j % chunkMax } +// pairFrom is the first full chunk (counted from 0) that is allocated in a +// pair; before it a tree stays small. +const pairFrom = 8 + +// appendChunk appends chunk k to cs. A full chunk is 20480 B, which the runtime +// pads to 21760 B once it adds its 8-byte malloc header; two together are 40960 +// B, a large object of exactly five pages with no header and no padding. The +// pair is allocated only if chunk k+1 is wanted (k < last), and each chunk is +// capped so it still looks like a separate allocation. +func appendChunk(cs [][]Node, k, last int) [][]Node { + if b := k - chunkMaxLog - 1; b >= pairFrom && b%2 == 0 && k < last { + s := make([]Node, 2*chunkMax) + return append(cs, s[:chunkMax:chunkMax], s[chunkMax:]) + } + return append(cs, make([]Node, chunkLen(k))) +} + // rec returns the record at index i. func (t *Tree) rec(i uint32) *Node { k, off := chunkOf(i) @@ -344,7 +361,7 @@ func (t *Tree) alloc() *Node { i := t.n k, off := chunkOf(i) if k == len(t.chunks) { - t.chunks = append(t.chunks, make([]Node, chunkLen(k))) + t.chunks = appendChunk(t.chunks, k, 1<<30) } t.n++ r := &t.chunks[k][off] From b4cd61475905b5858927ab12632c7e5346e49411 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:00:11 -0500 Subject: [PATCH 13/45] xdm: borrow source text blocks only for blocks a deep copy plans to fill by half --- xdm/record.go | 71 ++++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/xdm/record.go b/xdm/record.go index 506fbc0d..364a9517 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1685,9 +1685,9 @@ type textAux struct { // ws shares one stored value among the whitespace-only values add is // given, as a parse does. ws spaceTable - // lent counts, per block of another tree's store (keyed by address, which - // is only a count while the block is not borrowed), the bytes addFrom was - // asked to copy from it, and records where a borrowed block sits here. + // lent holds the blocks of another tree's store that planFrom named, keyed + // by address (a block is pinned by this store once it is borrowed, so its + // address cannot be reused while its entry says where it sits here). lent map[uintptr]*lendState // lastKey/last cache the previous lookup; copies run in document order. lastKey uintptr @@ -1695,8 +1695,7 @@ type textAux struct { } type lendState struct { - used int32 // bytes asked for from the block so far - idx int32 // index in this store of the borrowed block, + 1 (0: not borrowed) + idx int32 // index in this store of the borrowed block, + 1 (0: not borrowed yet) } // sharedKey identifies a string by where it lives: the same storage is the @@ -1777,31 +1776,33 @@ func (s *textStore) share(v string) { } // addFrom stores the value at (off, n) of src, the store of another tree, as -// add would store its string, but lets the range name src's block instead of -// a copy of the bytes once this store has been asked for half of the block's -// bytes: from then on every value of that block is free. A borrowed block is -// the source's slice with its capacity clipped to its length, so nothing is -// ever appended into it from here, and the source only appends past that -// length; the bytes below it never change. Borrowing pins the whole block, at -// worst twice the bytes this store asked of it, which bounds what a small -// result can retain of a large source. +// add would store its string, except that a block planFrom named is borrowed: +// the range names src's block instead of a copy of the bytes. A borrowed +// block is the source's slice with its capacity clipped to its length, so +// nothing is ever appended into it from here, and the source only appends past +// that length; the bytes below it never change. Borrowing pins the whole +// block, but planFrom names only blocks the copy is about to fill by half or +// more, so the pinned bytes are at most twice the copy's own. func (s *textStore) addFrom(src *textStore, off, n uint32) (uint32, uint32) { if n == 0 { return 0, 0 } - k := off >> textShift - blk := src.blocks[k] - // Whitespace goes through add's table; a value of a block of its own is - // kept by reference there already; and the store's block numbers must not - // run out for want of room to copy into. - if s == src || n >= textOwnFrom || len(s.blocks) >= 1<<(31-textShift) || - n <= maxSpaceLen && onlySpace(blk[off&textMask:][:n]) { + if s.aux == nil || len(s.aux.lent) == 0 || s == src || n >= textOwnFrom { + return s.add(src.str(off, n)) + } + blk := src.blocks[off>>textShift] + // Whitespace goes through add's table, as it does for a parse. + if n <= maxSpaceLen && onlySpace(blk[off&textMask:][:n]) { + return s.add(src.str(off, n)) + } + l := s.lend(blk, false) + if l == nil { return s.add(src.str(off, n)) } - l := s.lend(blk) if l.idx == 0 { - l.used += int32(n) - if 2*int(l.used) < len(blk) { + // The store's block numbers must not run out for want of room to copy + // into, so a store this full stops borrowing. + if len(s.blocks) >= 1<<(31-textShift) { return s.add(src.str(off, n)) } s.newBlock(blk[:len(blk):len(blk)]) @@ -1813,11 +1814,11 @@ func (s *textStore) addFrom(src *textStore, off, n uint32) (uint32, uint32) { return uint32(l.idx-1)< 0 && 2*int(c) >= len(blk) && len(blk) > 0 { - s.lend(blk).used = c + if blk := t.text.blocks[k]; c > 0 && 2*int(c) >= len(blk) { + s.lend(blk, true) } } } @@ -1843,10 +1844,13 @@ func (s *textStore) planFrom(t *Tree, lo, hi uint32) { // planMin is the fewest records planFrom walks. const planMin = 4096 -// lend returns the account of the block blk of another store. -func (s *textStore) lend(blk []byte) *lendState { +// lend returns the account of the block blk of another store, made if create. +func (s *textStore) lend(blk []byte, create bool) *lendState { a := s.aux if a == nil { + if !create { + return nil + } a = new(textAux) s.aux = a } @@ -1856,6 +1860,9 @@ func (s *textStore) lend(blk []byte) *lendState { } l := a.lent[key] if l == nil { + if !create { + return nil + } if a.lent == nil { a.lent = map[uintptr]*lendState{} } From 14a6c502c834f83934d747590426e9eb2491099f Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:05:31 -0500 Subject: [PATCH 14/45] xdm: say why paired full chunks may exceed the small-object cap --- xdm/record.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xdm/record.go b/xdm/record.go index 9d5aafd8..ec9e2f2d 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -309,7 +309,9 @@ func recentSlots(v *typingNames) uint64 { // The cap keeps every chunk a small-object allocation (at most 32 KB): a // larger one is allocated and freed as whole pages, which the runtime hands // back to the OS and faults in again, and on macOS that round trip costs more -// than the parse saves. +// than the parse saves. A large tree's full chunks are the exception: past +// pairFrom they come two to an allocation (see appendChunk), five pages that +// the malloc header would otherwise pad by 6%, and measured no slower. const ( chunkFirst = 4 chunkMaxLog = 7 // 4<<7 = 512 From 4cb3d136e15e4a77cfc29eb6727f4d2ddb675122 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:09:22 -0500 Subject: [PATCH 15/45] docs: record the footprint round CHANGELOG Unreleased; known-gaps: what the round left open and the one streamed shape; options: FileResolver.MaxNodes and MaxBytes reaching the parser; profiling: paired chunks against 508/511; test count 2,976. --- CHANGELOG.md | 37 +++++++++++++++++++++++++++ README.md | 9 ++++--- docs/conformance-gaps.md | 2 +- docs/known-gaps.md | 54 +++++++++++++++++++++++++++++++++++++--- docs/options.md | 7 +++++- docs/profiling.md | 1 + docs/stats.md | 2 +- docs/testing.md | 2 +- docs/todo.md | 2 +- 9 files changed, 104 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63d9558b..5b0175a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,30 @@ Notable changes, newest first. Versions follow [semantic versioning](https://semver.org): from 1.0.0 the exported API is stable, and a breaking change means 2.0 with a new module path. See *Stability* below. +## Unreleased + +Memory footprint work: what a tree holds and what a transform peaks at. Output +is byte-identical; new API is additive. + +### Added + +| Change | What it does | Commit | +|---|---|---| +| Streamed `xsl:source-document`, first stage | A streamable `xsl:for-each`/`xsl:iterate` over a child path reads one record at a time: 1 GB totalled at 27 MB RSS, not 3.7 GB, at about 2× CPU. Other shapes build the tree. | [`3f185fe`][3f185fe], [`27757b3`][27757b3] | +| `xdm.ParseBursts` | Parses a document handing over one selected element at a time in a tree of its own, with limits counted over the whole document. | [`b4b6052`][b4b6052], [`80f999f`][80f999f] | +| `xslt.FileResolver.MaxNodes`; `MaxBytes` reaches the parser | Raising `MaxBytes` past 64 MB had no effect on `fn:doc`, the parser kept its own default. Both limits now apply to what the resolver parses. | [`80f999f`][80f999f] | +| `xdmbuild.InPlaceDocument` | A builder rooted at a new document node, for content built straight into it. | [`302c932`][302c932] | + +### Changed — memory + +| Change | Problem → solution | Commit | +|---|---|---| +| Namespace nodes lived as long as their tree | `//namespace::*` on 5 MiB kept 140 MB. They are held weakly, 16 to a chunk, and freed when nothing holds them: under 0.5 MB after the query. | [`ac4973e`][ac4973e], [`fcab85a`][fcab85a], [`e1b5e1f`][e1b5e1f] | +| Built-in shallow-copy built a document result twice | In a mode with no template rules it builds in place: parse-100mb identity peak 1091 → 930 MB, CPU −25 to −40%. | [`302c932`][302c932] | +| Built trees stored every whitespace-only text value | They share one value per string, as a parse does: an identity result is the size of its source again. | [`c60f8f3`][c60f8f3], [`09f0c5d`][09f0c5d] | +| A deep copy duplicated the source's text | `xsl:copy-of` of 4,096+ records borrows the source blocks it fills by half or more: XMark identity result −28%, peak −19%. | [`546af03`][546af03], [`b4cd614`][b4cd614] | +| Full record chunks were padded 6% by the malloc header | They are allocated two at a time past the 8th, 40,960 B exactly: parse-100mb retained 370 → 350.5 MB. | [`907c674`][907c674] | + ## v2.1.1 — 2026-11-01 Memory and CPU fixes from go-xmlsec's measurements against v2.1.0. No API @@ -1830,3 +1854,16 @@ here so every entry in this file sits under a release. [9572c44]: https://github.com/knroy/go-xml/commit/9572c44 [fd9750e]: https://github.com/knroy/go-xml/commit/fd9750e [5712d9f]: https://github.com/knroy/go-xml/commit/5712d9f +[3f185fe]: https://github.com/knroy/go-xml/commit/3f185fe +[27757b3]: https://github.com/knroy/go-xml/commit/27757b3 +[b4b6052]: https://github.com/knroy/go-xml/commit/b4b6052 +[80f999f]: https://github.com/knroy/go-xml/commit/80f999f +[302c932]: https://github.com/knroy/go-xml/commit/302c932 +[ac4973e]: https://github.com/knroy/go-xml/commit/ac4973e +[fcab85a]: https://github.com/knroy/go-xml/commit/fcab85a +[e1b5e1f]: https://github.com/knroy/go-xml/commit/e1b5e1f +[c60f8f3]: https://github.com/knroy/go-xml/commit/c60f8f3 +[09f0c5d]: https://github.com/knroy/go-xml/commit/09f0c5d +[546af03]: https://github.com/knroy/go-xml/commit/546af03 +[b4cd614]: https://github.com/knroy/go-xml/commit/b4cd614 +[907c674]: https://github.com/knroy/go-xml/commit/907c674 diff --git a/README.md b/README.md index 2d5d00d9..4b1e03f7 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ and maintains it as a project of his own. | **XSD 1.1** | 99.98% instance (26,217 of 26,222); **99.97%** schema-validity (15,350 of 15,354); opt-in via `Version11` | | **RELAX NG** | 100.00% of James Clark's spectest (965 of 965 assertions); XML and compact syntax, from Go and from `go-xml validate -rng` | | **DTD** | content models, attribute defaults, enumerations, `ID`/`IDREF`; external subset, parameter entities across both subsets, conditional sections — via `dtd.Load` with a caller-supplied resolver, nothing fetched by default | -| **Tests** | 2,960 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | +| **Tests** | 2,976 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | | **Production schemas** | UBL 2.1, UN/CEFACT CII, Factur-X/ZUGFeRD, Peppol BIS 3.0 — 88 schemas load, instances validate clean | | **API** | 1.2; the exported surface is stable and additive over 1.1, and a breaking change means 2.0 with a new module path | @@ -178,7 +178,10 @@ true here: each test's declared version dependency — a different kind of measurement from running a suite written for the version under test. - **Streamed execution is not implemented**, but the streaming cases are in + **Streamed execution is implemented only for the narrowest shape** (an + `xsl:source-document streamable="yes"` whose body is one `xsl:for-each` or + `xsl:iterate` over a child path, read one record at a time; see + [known gaps](docs/known-gaps.md)). Otherwise the streaming cases are in scope and counted: the stylesheet-level half of streaming, the §19.8 streamability analysis that refuses a construct with `XTSE3430`, is implemented and measured. What is absent is streamed *execution*, not the @@ -1388,7 +1391,7 @@ back, is in [docs/testing.md](docs/testing.md). | method | what it catches | what it misses | |---|---|---| -| **Unit tests** (2,960 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** (2,976 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | | **Spec inventories** | features absent entirely | features present but behaving wrongly | | **Saxon differential** | subtle behavioural divergence on real stylesheets | constructs the corpora do not use | | **W3C QT3 suite** | systematic conformance across 22,054 XPath and 30,517 XQuery cases | XSLT (it is an XPath suite) | diff --git a/docs/conformance-gaps.md b/docs/conformance-gaps.md index b963d935..4d10d7f5 100644 --- a/docs/conformance-gaps.md +++ b/docs/conformance-gaps.md @@ -49,7 +49,7 @@ W3C disagreements: 0 + 0 + 0 + 1 + 8 + 23 + 30 + 31 + 0 = 93. Measured 2026-09-1 -The unit-test suite is 2,960 tests. +The unit-test suite is 2,976 tests. The last two rows are not W3C suites but real-world corpora — DocBook xslTNG's diff --git a/docs/known-gaps.md b/docs/known-gaps.md index a4001b8b..316b19a2 100644 --- a/docs/known-gaps.md +++ b/docs/known-gaps.md @@ -128,9 +128,10 @@ Found by the third memory audit (2026-11-01) and not fixed, each with the reason (40k items 12.4 GB, all garbage). Appending in place is unsound while other references may share the prefix. - **Identity templates allocate about 3.5 KB a node of context copies** - (`withSel`, `hostWithCurrent`, the leaf-hook copy), and built-in shallow-copy - copies a document-level result twice; removing that copy changes an error - case into output. + (`withSel`, `hostWithCurrent`, the leaf-hook copy). Built-in shallow-copy of + a document node builds the result once in a mode with no template rules + (v2.2.0), but still builds it and copies it into a document in a mode that + has rules, where a rule could add an attribute that must stay `XTDE0420`. - **Several large text values in one document cost about 1.3× their size to parse** (eight 3 MiB values: 1.30×). A single value is about 1.1× at every size since v2.1.1; with several, the first value's buffer overshoots once @@ -150,6 +151,50 @@ Found by the third memory audit (2026-11-01) and not fixed, each with the reason - **A regex program over 4,096 instructions is compiled on every call**, because it is not cached; the largest QT3 pattern is 4,059. +### Memory: what the footprint round left open (performance) + +Measured on branches in 2026-11 and not adopted, each with the reason: + +- **A 32-byte node record** (the tree pointer read from a chunk header): + −11 to −22% retained, rejected because a by-value copy of an `xdm.Node` + would then read foreign memory instead of failing safely. +- **Compact attribute records** (16 B, nodes made on demand): −10% retained on + parse-100mb, −16% on SAML, about 0 on XMark, but +10–25% CPU on + attribute-heavy XPath, c14n and copies (+45% worst case), and a held + attribute would cost 62 B instead of 40. Pointer identity of `*xdm.Node` + forces a real record per held node; denser storage needs a value-type node + handle, which is a breaking change. +- **Template-driven shallow-copy does not share source text.** Deep copies + (`xsl:copy-of`, `xdm.Copy` of 4,096+ records) borrow the source's text + blocks; a template-built copy adds each value as a string. +- **Holding millions of namespace nodes costs a sweep per collection.** + Namespace nodes are held weakly since v2.2.0; while a tree's table is not + empty, every GC cycle sweeps it. + +### Streamed execution covers one shape (XSLT 3.0, performance) + +Since v2.2.0 an `xsl:source-document streamable="yes"` whose body is one +`xsl:for-each` or `xsl:iterate` (optionally inside constant literal result +elements) over a child path of plain names, with non-positional predicates on +the last step only, and a body the gate recognises, runs record by record +through `xdm.ParseBursts`: a 1 GB file totalled by `xsl:iterate` peaks at +27 MB RSS against 3.7 GB, at about 2× the CPU. Everything else builds the tree, +as before. In the W3C suite 9 of the 2,599 streamable source documents take the +burst path; the commonest refusals are a body of `xsl:value-of` (840), +`xsl:copy-of` (344) or `xsl:apply-templates` (175). Also: + +- **The result is still built in memory**, so a large filtered copy grows with + its output and stops at the result-node cap. +- **Error order can differ from tree mode.** A dynamic error in the body is + raised before a malformed tail later in the file would be; tree mode reads + the whole file first. `xsl:try` can see which code wins. +- **`xsl:break` still reads the rest of the file**, so that a malformed tail is + an error as in tree mode. +- **Only `FileResolver` streams**; another `DocumentResolver` builds the tree. + The streamed document is not cached, so a later `fn:doc` of it reads it again. +- **The `FileResolver` limits apply to the whole file**: raise `MaxBytes` and + `MaxNodes` to stream a document past 64 MB or 10M nodes. + ### `fn:replace` charges a backtracking or backreference result after building it (XPath, resource limits) The RE2 path of `fn:replace` charges `len(input) + matches × len(replacement)` @@ -179,7 +224,8 @@ and §18.1's grounded demand at both the sites it names. All three fail in the same direction: the suite expects `XTSE3430` — *this construct is not guaranteed streamable* — and the transform succeeds instead. -The engine builds a tree and streams nothing, so it produces the **right +The engine builds a tree for these (it reads in bursts only the narrow shape +described under *Streamed execution covers one shape*), so it produces the **right answer** for all three; what it does not produce is the static refusal §19.8 requires a streaming processor to make before running anything. These are not wrong answers; they are a static analysis that is not complete. diff --git a/docs/options.md b/docs/options.md index 7701fffa..92a947bc 100644 --- a/docs/options.md +++ b/docs/options.md @@ -837,7 +837,8 @@ only the roots. | `AllowDOCTYPE` | `bool` | refuse | Permits a `DOCTYPE` in the documents this resolver parses. | | `ExternalEntities` | `bool` | refuse | Permits those documents to read external entities, through this same resolver. Separate from `AllowDOCTYPE`. | | `UnparsedText` | `bool` | refuse | Permits `fn:unparsed-text` to read through this resolver. | -| `MaxBytes` | `int64` | `DefaultMaxResourceBytes` = 64 MB | Bounds **one file**, whichever way it is asked for. Zero means the default; negative means no limit. A file over the limit is refused, naming it — never truncated. | +| `MaxBytes` | `int64` | `DefaultMaxResourceBytes` = 64 MB | Bounds **one file**, whichever way it is asked for. Zero means the default; negative means no limit. A file over the limit is refused, naming it — never truncated. Since v2.2.0 it is also the parser's byte limit for the documents this resolver parses. | +| `MaxNodes` | `int` | `xdm.DefaultMaxNodes` = 10M | Bounds the nodes of one document this resolver parses, as `xdm.ParseOptions.MaxNodes` does. Zero means the default; negative means no limit. Counted over the whole document even when a streamable `xsl:source-document` reads it record by record. | `MaxBytes` is one number rather than one per call path deliberately. The confinement is a property of the *file*: every root is readable by every path, @@ -847,6 +848,10 @@ is the larger of them. It is set to the same 64 MB as `xdm.DefaultMaxBytes`, so that a document refused at the read is refused before the bytes are spent and the parser's identical limit stays the backstop for input arriving another way. +Before v2.2.0 the parser kept its own 64 MB default whatever `MaxBytes` said, +so raising `MaxBytes` did not let a larger document through `fn:doc`; it now +does. + It bounds what `xdm.ParseOptions.MaxBytes` cannot. That limit bounds the *parse*; the resolver has the whole file in memory before the parser is handed anything, and `fn:unparsed-text` and XInclude `parse="text"` never reach a diff --git a/docs/profiling.md b/docs/profiling.md index 181d57e5..39e469db 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -482,6 +482,7 @@ longer holds. | Interning `nodeTyping` per tree | Bytes −27%, wall +25%: hashing the key on every typing write costs more than the allocation saves | | Typing chunks under 32 KB | No gain. The `madvise` time in the typed path is macOS heap growth, not chunk size | | Bigger node chunks; 1,024-node chunks | No gain; +0.28 GB peak RSS at 100 MB | +| 508- or 511-record full chunks, to fit the 20,480 B size class (2026-11) | −4.6 to −5.1% retained on parse-100mb, but `chunkOf` divides instead of shifting: tree walk 1.32–1.46×, child name test 1.20–1.31×. **Adopted instead:** 512-record chunks allocated two at a time past the 8th full chunk (`appendChunk`), 40,960 B with no malloc header to pad: −5.3% retained and peak on parse-100mb, CPU within noise | | Smaller first record chunk (2 records, V15) | Most fragments hold one node, but bytes moved −0.4% to +0.2% and allocations up to +0.6%: the next chunk comes sooner for every other tree | | Chunked result nodes | −0.9% allocations at most, +0.4–0.7% bytes | | String arena starting small | Retained heap −20–37% on small documents, but DocBook +5% CPU: the smaller live heap ran the collector more often (85 cycles against 77). Reverted in `629cf48` | diff --git a/docs/stats.md b/docs/stats.md index f5baea86..64e66c53 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -23,7 +23,7 @@ claim is only as good as the command behind it. | figure | count | counted by | |---|---:|---| -| Unit tests | 2,960 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | +| Unit tests | 2,976 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Fuzz targets | 12 | `grep -rn '^func Fuzz' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Limit boundary tests | 14 | `grep -hc '^func Test' ./*/limits_boundary*_test.go \| awk '{n += $1} END {print n + 0}'` | diff --git a/docs/testing.md b/docs/testing.md index 8fd05879..86f81108 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,7 +21,7 @@ let something through, and the column that matters is the last one. | layer | count | catches | misses | |---|---:|---|---| -| **Unit tests** | 2,960 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** | 2,976 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | | **Limit boundary tests** | 14 tests | an off-by-one or an overflow at the edge of a configurable limit | a limit nobody added to the inventory | | **Race detector** | same tests | shared state a single-goroutine run never reveals | a data race on a path no test walks | | **W3C conformance suites** | 152,241 cases | systematic divergence from the specification | what the suites do not ask about — see below | diff --git a/docs/todo.md b/docs/todo.md index 0a598686..d53e80a9 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -23,7 +23,7 @@ Current position: | XSLT 3.0 | 99.80% — 11,495 of 11,518 in scope (23 failing); 3 of those need more of the §19.8 streamability analysis | | RELAX NG | 100.00% — 965 of 965 | | Schemas wrongly refused | 7 — 6 on XSD 1.0, 1 on 1.1 | -| Tests | 2,960 `func Test` declarations, clean under `-race` | +| Tests | 2,976 `func Test` declarations, clean under `-race` | Every one of those failures, and why it is still open, is catalogued in [known-gaps.md](known-gaps.md). This file is the forward-looking half — what From 36a8e757568d1ab6c4e2c0c9a624db20fce237e2 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:33:03 -0500 Subject: [PATCH 16/45] xdm: truncate takes sideMu; an emptied side table stays while a sweep is pending (M4, L1) --- xdm/build.go | 4 +--- xdm/side_sweep_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++ xdm/sideweak.go | 14 +++++++++++- 3 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 xdm/side_sweep_test.go diff --git a/xdm/build.go b/xdm/build.go index 7d725d46..d6728777 100644 --- a/xdm/build.go +++ b/xdm/build.go @@ -268,9 +268,7 @@ func (t *Tree) truncate(i uint32) { delete(t.source.foreignPos, j) } } - if t.side != nil { - t.sideTruncate(i) - } + t.sideTruncate(i) // locks: the sweep cleanup writes t.side t.n = i } diff --git a/xdm/side_sweep_test.go b/xdm/side_sweep_test.go new file mode 100644 index 00000000..20dadce0 --- /dev/null +++ b/xdm/side_sweep_test.go @@ -0,0 +1,51 @@ +package xdm + +import ( + "runtime" + "testing" + "time" +) + +// nsAxis makes the namespace nodes of a fresh child of r and takes it back. +func nsAxis(r *Node) { + e := r.AppendElement(QName{Local: "e"}) + for range e.NamespaceNodes() { + } + r.RemoveLastChild() +} + +// TestSideTruncateRacesSweep: RemoveLastChild reads the namespace table while +// the sweep cleanup goroutine clears it; under -race an unlocked read fails. +func TestSideTruncateRacesSweep(t *testing.T) { + tree := NewTree() + r := tree.Root.AppendElement(QName{Local: "r"}) + nsAxis(r) + runtime.GC() + time.Sleep(50 * time.Millisecond) // no happens-before edge for the detector + r.AppendElement(QName{Local: "e"}) // no namespace query: nothing locks sideMu first + r.RemoveLastChild() +} + +// TestSideTruncateKeepsOneSweepChain: emptying the table while a sweep is +// pending must not let the next add arm a second chain. +func TestSideTruncateKeepsOneSweepChain(t *testing.T) { + tree := NewTree() + r := tree.Root.AppendElement(QName{Local: "r"}) + for i := 0; i < 10; i++ { + nsAxis(r) + } + e := r.AppendElement(QName{Local: "e"}) + var keep []*Node + for ns := range e.NamespaceNodes() { + keep = append(keep, ns) // the table stays non-empty, so chains re-arm + } + before := sideSweeps.Load() + for i := 0; i < 5; i++ { + runtime.GC() + time.Sleep(20 * time.Millisecond) + } + if n := sideSweeps.Load() - before; n > 7 { + t.Errorf("%d sweeps in 5 collections (want about 5): more than one chain", n) + } + runtime.KeepAlive(keep) +} diff --git a/xdm/sideweak.go b/xdm/sideweak.go index cda6cde5..4696b215 100644 --- a/xdm/sideweak.go +++ b/xdm/sideweak.go @@ -2,9 +2,14 @@ package xdm import ( "runtime" + "sync/atomic" "weak" ) +// sideSweeps counts the sweeps that have run, for the test that checks a tree +// keeps one sweep chain. +var sideSweeps atomic.Int64 + // sideChunkLen is how many namespace nodes share one allocation, and so one // weak handle: the runtime keeps a list per heap span of every object with a // weak handle, and a handle per node made a namespace-heavy query several @@ -85,6 +90,7 @@ type sideTick struct{ _ *int } // sideSweep drops the entries whose nodes a collection has freed, and gives // the memory of a table that has shrunk a lot back (a Go map never does). func sideSweep(t *Tree) { + sideSweeps.Add(1) t.sideMu.Lock() defer t.sideMu.Unlock() st := t.side @@ -134,6 +140,12 @@ func (t *Tree) sideTruncate(i uint32) { } } if len(t.side.m) == 0 { - t.side = nil + // While a sweep is pending the empty table stays, or the next + // sideAdd would arm a second chain; the sweep drops it. + if t.side.armed { + t.side.seq = 0 + } else { + t.side = nil + } } } From 696bee46275d0fb4cbc8852b687dac5e8d44e8ec Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:33:28 -0500 Subject: [PATCH 17/45] xdm: planFrom compares against the block's capacity, which borrowing pins (L3) --- xdm/lend_pin_test.go | 36 ++++++++++++++++++++++++++++++++++++ xdm/record.go | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 xdm/lend_pin_test.go diff --git a/xdm/lend_pin_test.go b/xdm/lend_pin_test.go new file mode 100644 index 00000000..e89b4109 --- /dev/null +++ b/xdm/lend_pin_test.go @@ -0,0 +1,36 @@ +package xdm + +import ( + "strings" + "testing" +) + +// TestPlanFromCountsCapacity: a copy borrows a source block only when the text +// it takes is at least half the array it would pin, and an untrimmed built +// block is 32 KB however little it holds. +func TestPlanFromCountsCapacity(t *testing.T) { + bt := NewTree() + br := bt.Root.AppendElement(QName{Local: "r"}) + for i := 0; i < 70; i++ { + br.AppendElement(QName{Local: "p"}).AppendText(strings.Repeat("w", 1000)) + } + for i := 0; i < 8; i++ { + br.AppendElement(QName{Local: "p"}).AppendText(strings.Repeat("y", 8000)) + } + sub := br.AppendElement(QName{Local: "s"}) + sub.AppendText(strings.Repeat("z", 8000)) + for i := 0; i < planMin; i++ { + sub.AppendElement(QName{Local: "e"}) + } + last := bt.text.blocks[len(bt.text.blocks)-1] + if len(last) >= cap(last)/2 || len(last) < 2*8000 { + t.Fatalf("setup: last block len %d cap %d", len(last), cap(last)) + } + dst := NewTree() + dst.Root.AppendCopy(sub) + for _, b := range dst.text.blocks { + if len(b) > 0 && &b[0] == &last[0] { + t.Errorf("copy borrowed the source block: pins %d B for 8000 B copied", cap(last)) + } + } +} diff --git a/xdm/record.go b/xdm/record.go index c9d1062c..5d922b68 100644 --- a/xdm/record.go +++ b/xdm/record.go @@ -1854,7 +1854,7 @@ func (s *textStore) planFrom(t *Tree, lo, hi uint32) { } } for k, c := range counts { - if blk := t.text.blocks[k]; c > 0 && 2*int(c) >= len(blk) { + if blk := t.text.blocks[k]; c > 0 && 2*int(c) >= cap(blk) { // borrowing pins the array s.lend(blk, true) } } From 4518aea122f850b9d7439bf14ca091568c046d86 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:35:21 -0500 Subject: [PATCH 18/45] xdm: spaceTable makes its length table after eight values, so a small tree does not pay a kilobyte (M3) --- xdm/parse.go | 11 ++++++++++- xdm/spacetable_test.go | 27 ++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/xdm/parse.go b/xdm/parse.go index 48c7c136..321d7092 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -1016,7 +1016,8 @@ func (r *textRun) flush(spaces *spaceTable, strip func(*Node) bool) { // every repeat. type spaceTable struct { m map[string][2]uint32 - byLen [maxSpaceLen + 1][2]uint32 + byLen *[maxSpaceLen + 1][2]uint32 // made after spaceFrom values, so a small tree does not pay for it + n uint8 // values seen before byLen was made // noMap keeps m unmade: ParseBursts empties the table at every burst. noMap bool } @@ -1024,12 +1025,20 @@ type spaceTable struct { const ( maxSpaceTable = 256 maxSpaceLen = 128 + spaceFrom = 8 // whitespace values stored plainly before sharing starts ) func (t *spaceTable) text(s *textStore, b []byte) (uint32, uint32) { if len(b) > maxSpaceLen || !onlySpace(b) { return s.addBytes(b, 0) } + if t.byLen == nil { + if t.n < spaceFrom { + t.n++ + return s.addBytes(b, 0) + } + t.byLen = new([maxSpaceLen + 1][2]uint32) + } if r := t.byLen[len(b)]; r[1] != 0 && s.str(r[0], r[1]) == string(b) { return r[0], r[1] } diff --git a/xdm/spacetable_test.go b/xdm/spacetable_test.go index 5382d75e..8c91eccf 100644 --- a/xdm/spacetable_test.go +++ b/xdm/spacetable_test.go @@ -1,6 +1,9 @@ package xdm -import "testing" +import ( + "runtime" + "testing" +) // Whitespace runs are shared by length first; two runs of one length but // different characters must still keep their own values. @@ -34,3 +37,25 @@ func TestSpaceTableSameLengthRuns(t *testing.T) { } } } + +// TestSmallTreeSkipsSpaceTable: the first whitespace value in a small tree +// must not cost the table's kilobyte (a built tree holding " " was 2.2 KB). +func TestSmallTreeSkipsSpaceTable(t *testing.T) { + const n = 200 + build := func() *Tree { + tr := NewTree() + tr.Root.AppendElement(QName{Local: "a"}).AppendText(" ") + tr.Finalize() + return tr + } + build() + var a, b runtime.MemStats + runtime.ReadMemStats(&a) + for i := 0; i < n; i++ { + build() + } + runtime.ReadMemStats(&b) + if per := (b.TotalAlloc - a.TotalAlloc) / n; per > 1500 { + t.Errorf("a tree holding one space allocated %d B, want under 1500", per) + } +} From e8e84939fc754565baded1658f8548adf23add64 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:35:21 -0500 Subject: [PATCH 19/45] xdm: gofmt side_sweep_test.go --- xdm/side_sweep_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xdm/side_sweep_test.go b/xdm/side_sweep_test.go index 20dadce0..8fd33474 100644 --- a/xdm/side_sweep_test.go +++ b/xdm/side_sweep_test.go @@ -21,7 +21,7 @@ func TestSideTruncateRacesSweep(t *testing.T) { r := tree.Root.AppendElement(QName{Local: "r"}) nsAxis(r) runtime.GC() - time.Sleep(50 * time.Millisecond) // no happens-before edge for the detector + time.Sleep(50 * time.Millisecond) // no happens-before edge for the detector r.AppendElement(QName{Local: "e"}) // no namespace query: nothing locks sideMu first r.RemoveLastChild() } From 1f48b3130585b354d35628bc281a88c2b6996e91 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:39:40 -0500 Subject: [PATCH 20/45] xdm: spaceTable keeps its first four distinct values inline, so tiny and small trees still share whitespace (M3) --- xdm/parse.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/xdm/parse.go b/xdm/parse.go index 321d7092..a3deffd9 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -1016,8 +1016,9 @@ func (r *textRun) flush(spaces *spaceTable, strip func(*Node) bool) { // every repeat. type spaceTable struct { m map[string][2]uint32 - byLen *[maxSpaceLen + 1][2]uint32 // made after spaceFrom values, so a small tree does not pay for it - n uint8 // values seen before byLen was made + byLen *[maxSpaceLen + 1][2]uint32 // made past few, so a small tree does not pay for it + few [spaceFew][2]uint32 // the first distinct values, until byLen is made + nFew uint8 // noMap keeps m unmade: ParseBursts empties the table at every burst. noMap bool } @@ -1025,7 +1026,7 @@ type spaceTable struct { const ( maxSpaceTable = 256 maxSpaceLen = 128 - spaceFrom = 8 // whitespace values stored plainly before sharing starts + spaceFew = 4 // distinct values shared before the length table is made ) func (t *spaceTable) text(s *textStore, b []byte) (uint32, uint32) { @@ -1033,9 +1034,16 @@ func (t *spaceTable) text(s *textStore, b []byte) (uint32, uint32) { return s.addBytes(b, 0) } if t.byLen == nil { - if t.n < spaceFrom { - t.n++ - return s.addBytes(b, 0) + for _, r := range t.few[:t.nFew] { + if s.str(r[0], r[1]) == string(b) { + return r[0], r[1] + } + } + if t.nFew < spaceFew { + off, n := s.addBytes(b, 0) + t.few[t.nFew] = [2]uint32{off, n} + t.nFew++ + return off, n } t.byLen = new([maxSpaceLen + 1][2]uint32) } From 585b7159779dc5ddda524004e20ec354c5e8bf23 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:39:57 -0500 Subject: [PATCH 21/45] xdm: a clone does not copy the name and frame indexes ParseBursts cloned the parse tree's indexes into every burst, and they grow with the stream, so CPU was quadratic in distinct names and namespace declarations. intern and internFrame rebuild them on use. --- xdm/bursts_test.go | 31 +++++++++++++++++++++++++++++++ xdm/clone.go | 9 ++++----- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/xdm/bursts_test.go b/xdm/bursts_test.go index 08c7e097..ba0de419 100644 --- a/xdm/bursts_test.go +++ b/xdm/bursts_test.go @@ -140,3 +140,34 @@ func TestParseBurstsSpineStaysSmall(t *testing.T) { t.Fatalf("the spine kept %d records and %d text bytes", tree.n, bytes) } } + +// A burst's tree does not carry the parse's name and frame indexes, which +// cover every name and declaration of the stream so far: copying them made +// the cost of each burst grow with the stream. Its names still resolve. +func TestParseBurstsCopyNoIndexes(t *testing.T) { + var b strings.Builder + b.WriteString("") + for i := range 200 { + fmt.Fprintf(&b, ``, i, i, i) + } + b.WriteString("") + i := 0 + err := ParseBursts(strings.NewReader(b.String()), ParseOptions{}, []QName{{Local: "a"}, {Local: "r"}}, + func(r *Node) error { + c := r.tree + if c.nameIx != nil || c.frameOne != nil || c.frameMany != nil { + t.Fatalf("burst %d: copied indexes: %d names, %d+%d frames", i, len(c.nameIx), len(c.frameOne), len(c.frameMany)) + } + if got, want := r.FirstChild().Name().Local, fmt.Sprintf("n%d", i); got != want { + t.Fatalf("burst %d: child %q, want %q", i, got, want) + } + if got := c.intern(QName{Local: fmt.Sprintf("n%d", i)}); got != r.FirstChild().name { + t.Fatalf("burst %d: intern finds %d, the child has %d", i, got, r.FirstChild().name) + } + i++ + return nil + }) + if err != nil || i != 200 { + t.Fatalf("%d bursts, %v", i, err) + } +} diff --git a/xdm/clone.go b/xdm/clone.go index 67f6a0aa..784aad69 100644 --- a/xdm/clone.go +++ b/xdm/clone.go @@ -1,8 +1,6 @@ package xdm import ( - "maps" - "github.com/knroy/go-xml/v2/internal/xdmclone" ) @@ -112,8 +110,11 @@ func cloneSubtree(top *Node, o xdmclone.Options) func(*Node) *Node { if layer != nil && layer.src != t { layer = nil } + // The name and frame indexes are not copied: intern and internFrame + // rebuild them on the copy's first use, as for a frozen source, which + // has none. A burst's copy is made from a tree still being parsed, whose + // indexes cover every name and frame of the stream so far. c.names = t.names[:len(t.names):len(t.names)] - c.nameIx = maps.Clone(t.nameIx) c.text.blocks = make([][]byte, len(t.text.blocks)) for i, b := range t.text.blocks { c.text.blocks[i] = b[:len(b):len(b)] @@ -121,8 +122,6 @@ func cloneSubtree(top *Node, o xdmclone.Options) func(*Node) *Node { c.text.size = t.text.size c.frames = t.frames[:len(t.frames):len(t.frames)] c.frameData = t.frameData[:len(t.frameData):len(t.frameData)] - c.frameOne = maps.Clone(t.frameOne) - c.frameMany = maps.Clone(t.frameMany) total := n if newIdx != nil { From f234c6297b75c22be09d45ac165a5ba8f21c5df7 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:40:32 -0500 Subject: [PATCH 22/45] xdm: a burst parse keeps no prolog comments or PIs Every burst copied them, so the cost was prolog x records. No burst body can reach the document node's other children. --- xdm/bursts.go | 8 +++++--- xdm/bursts_test.go | 25 +++++++++++++++++++++++++ xdm/parse.go | 6 ++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/xdm/bursts.go b/xdm/bursts.go index b100e622..e7fa89aa 100644 --- a/xdm/bursts.go +++ b/xdm/bursts.go @@ -23,7 +23,8 @@ import ( // declarations, attributes, base URI) and a deep copy of the element. So the // element's base URI, in-scope namespaces, attributes, parent and ancestors // read as they would in the whole tree; its siblings, and anything else -// outside its ancestry, do not exist there. Each burst is a new tree, so +// outside its ancestry -- the comments and processing instructions outside +// the document element too -- do not exist there. Each burst is a new tree, so // nodes from different bursts are distinct and order by creation, which is // document order; a node fn keeps stays valid after fn returns. @@ -118,8 +119,9 @@ func (b *burster) discardLast(p *Node, spaces *spaceTable) { } // deliver hands el to fn in a tree of its own. By now the parse's tree holds -// nothing but the document node, any prolog comments and processing -// instructions, the spine and el, which is exactly the burst's tree: it is +// nothing but the document node, the spine and el (a burst parse does not +// keep the prolog's comments and processing instructions, which every burst +// would copy), which is exactly the burst's tree: it is // copied in bulk, sharing the names, frames and text the records point into. func (b *burster) deliver(el *Node) error { m := cloneSubtree(el.tree.Root, xdmclone.Options{}) diff --git a/xdm/bursts_test.go b/xdm/bursts_test.go index ba0de419..5f9aa5f9 100644 --- a/xdm/bursts_test.go +++ b/xdm/bursts_test.go @@ -171,3 +171,28 @@ func TestParseBurstsCopyNoIndexes(t *testing.T) { t.Fatalf("%d bursts, %v", i, err) } } + +// The comments and processing instructions around the document element are +// not copied into every burst, which made the cost prolog x records. +func TestParseBurstsNoProlog(t *testing.T) { + src := strings.Repeat("", 100) + "" + strings.Repeat("", 50) + "" + n := 0 + err := ParseBursts(strings.NewReader(src), ParseOptions{}, []QName{{Local: "a"}, {Local: "r"}}, + func(r *Node) error { + n++ + // The document node, a, r and x. + if got := r.tree.n; got != 4 { + t.Fatalf("burst %d: the tree holds %d records, want 4", n, got) + } + return nil + }) + if err != nil || n != 50 { + t.Fatalf("%d bursts, %v", n, err) + } + // Still checked: a PI target with a colon in the prolog. + err = ParseBursts(strings.NewReader(""), ParseOptions{}, []QName{{Local: "a"}, {Local: "r"}}, + func(*Node) error { return nil }) + if err == nil { + t.Fatal("a colon in a prolog PI target: no error") + } +} diff --git a/xdm/parse.go b/xdm/parse.go index 48c7c136..b74e40d0 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -505,6 +505,9 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { case *xml.Comment: t := *tok sawPrologToken = true + if bs != nil && cur == tree.Root { + continue // a burst's tree holds no prolog (see bursts.go) + } c := cur.appendChild(KindComment) c.v0, c.v1 = tree.text.addBytes(t, 0) @@ -529,6 +532,9 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { return nil, fmt.Errorf("parse XML: processing-instruction target %q contains a colon", t.Target) } sawPrologToken = true + if bs != nil && cur == tree.Root { + continue + } pi := cur.appendChild(KindPI) pi.name = tree.intern(QName{Local: t.Target}) pi.v0, pi.v1 = tree.text.addBytes(t.Inst, 0) From 7e07cd6da59db37a882eccda870fa21357950774 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:42:44 -0500 Subject: [PATCH 23/45] xdm: a burst parse takes back the names and frames it drops The parse's name and frame tables grew with the whole stream. Past 1024 new ones beyond the spine they are rewound like the text store, clipped so the bursts that share them keep reading theirs. --- xdm/bursts.go | 103 +++++++++++++++++++++++++++++++++++++++------ xdm/bursts_test.go | 38 +++++++++++++++++ xdm/parse.go | 6 +-- 3 files changed, 132 insertions(+), 15 deletions(-) diff --git a/xdm/bursts.go b/xdm/bursts.go index e7fa89aa..d6753d00 100644 --- a/xdm/bursts.go +++ b/xdm/bursts.go @@ -49,13 +49,24 @@ type burster struct { // drop is the depth of the element being built only to be dropped (a // burst, or an element off the path), or 0. drop int - // marks[d] is the text store's state once the spine element at depth d - // (0: the document node) had stored its attributes. Everything stored - // after it belongs to content that is dropped. - marks []textMark + // marks[d] is the tree's state once the spine element at depth d (0: + // the document node) had stored its attributes. Everything stored after + // it belongs to content that is dropped. + marks []burstMark v11 bool } +// burstMark is how much of the text store, the name table and the frame +// table a spine element and its ancestors use. +type burstMark struct { + text textMark + names, frames, frameData int +} + +func markOf(t *Tree) burstMark { + return burstMark{t.text.mark(), len(t.names), len(t.frames), len(t.frameData)} +} + // start classifies the element just built at depth d and opened as el. func (b *burster) start(el *Node, d int) { if b.drop != 0 { @@ -64,7 +75,7 @@ func (b *burster) start(el *Node, d int) { if d == b.spine+1 && b.matches(el, d) { if d < len(b.path) { b.spine = d - b.marks = append(b.marks, el.tree.text.mark()) + b.marks = append(b.marks, markOf(el.tree)) return } } @@ -78,7 +89,7 @@ func (b *burster) matches(el *Node, d int) bool { // end handles the close of el, at depth d, whose parent is now current. It // delivers a burst and drops whatever is not spine. -func (b *burster) end(el *Node, d int, spaces *spaceTable) error { +func (b *burster) end(el *Node, d int, spaces *spaceTable, names *nameCache) error { switch { case b.drop == d: b.drop = 0 @@ -93,26 +104,30 @@ func (b *burster) end(el *Node, d int, spaces *spaceTable) error { default: return nil } - b.discardLast(el.Parent(), spaces) + b.discardLast(el.Parent(), spaces, names) return nil } // trimLeaves drops the leaves completed directly inside the spine element // cur, at depth d: text, comments and processing instructions no path step // selects. -func (b *burster) trimLeaves(cur *Node, d int, spaces *spaceTable) { +func (b *burster) trimLeaves(cur *Node, d int, spaces *spaceTable, names *nameCache) { if b.drop != 0 || d == 0 || d != b.spine { return } for last := cur.LastChild(); last != nil && last.isLeaf(); last = cur.LastChild() { - b.discardLast(cur, spaces) + b.discardLast(cur, spaces, names) } } -// discardLast removes p's last child and the text stored since p's spine mark. -func (b *burster) discardLast(p *Node, spaces *spaceTable) { +// discardLast removes p's last child and the text, names and namespace +// frames stored since p's spine mark; names is the parse's name cache. +func (b *burster) discardLast(p *Node, spaces *spaceTable, names *nameCache) { p.RemoveLastChild() - p.tree.text.rewind(b.marks[len(b.marks)-1]) + m := b.marks[len(b.marks)-1] + p.tree.text.rewind(m.text) + rewindNames(p.tree, m.names, names) + p.tree.rewindFrames(m.frames, m.frameData) // The whitespace table remembers ranges in the store, and some of them // were just released. Its map is not rebuilt for the next burst. *spaces = spaceTable{noMap: true} @@ -157,3 +172,67 @@ func (s *textStore) rewind(m textMark) { s.aux.lent, s.aux.last = nil, nil } } + +// burstSlack is how many names, or frames, the content of the spine may add +// before they are taken back. Most documents repeat a few names, which are +// then interned once rather than again for every burst; distinct ones are +// held to this many. +const burstSlack = 1024 + +// rewindNames forgets the names interned since the table held n, and their +// indices in the tree's cache and in cache, once there are more than +// burstSlack. A burst's +// tree shares the table, so the table is clipped rather than overwritten: +// the next name appended copies it. +func rewindNames(t *Tree, n int, cache *nameCache) { + if len(t.names) <= n+burstSlack { + return + } + for _, q := range t.names[n:] { + if i, ok := t.nameIx[q]; ok && int(i) >= n { + delete(t.nameIx, q) + } + } + t.names = t.names[:n:n] + if n == 0 { + t.names = nil // intern starts a table with the empty name at 0 + } + if h := t.nameHot; h != nil { + for i, x := range h { + if int(x) >= n { + h[i] = 0 + } + } + } + for i := range cache { + if int(cache[i].idx) >= n { + cache[i].q, cache[i].idx = QName{}, 0 + } + } +} + +// rewindFrames forgets the namespace frames interned since the table held n +// frames and nd bindings, as rewindNames does names. +func (t *Tree) rewindFrames(n, nd int) { + if len(t.frames) <= n+burstSlack { + return + } + for i := n; i < len(t.frames); i++ { + f := t.frameAt(uint32(i)) + if len(f) == 1 { + if j, ok := t.frameOne[f[0]]; ok && int(j) >= n { + delete(t.frameOne, f[0]) + } + } else if t.frameMany != nil { + k := frameKey(f) + if j, ok := t.frameMany[k]; ok && int(j) >= n { + delete(t.frameMany, k) + } + } + } + t.frames = t.frames[:n:n] + if n == 0 { + t.frames = nil + } + t.frameData = t.frameData[:nd:nd] +} diff --git a/xdm/bursts_test.go b/xdm/bursts_test.go index 5f9aa5f9..8f697df4 100644 --- a/xdm/bursts_test.go +++ b/xdm/bursts_test.go @@ -196,3 +196,41 @@ func TestParseBurstsNoProlog(t *testing.T) { t.Fatal("a colon in a prolog PI target: no error") } } + +// The names and namespace frames that the dropped content interns are taken +// back as the stream goes, as its text is, and the bursts kept from before +// still read theirs. +func TestParseBurstsNamesStaySmall(t *testing.T) { + const n = 5000 + var b strings.Builder + b.WriteString("") + for i := range n { + fmt.Fprintf(&b, ``, i, i, i, i, i, i) + } + b.WriteString("") + var got []*Node + opts := ParseOptions{bursts: &burster{ + path: []QName{{Local: "a"}, {Local: "r"}}, + fn: func(r *Node) error { got = append(got, r); return nil }, + }} + tree, err := parse(strings.NewReader(b.String()), "", opts) + if err != nil { + t.Fatal(err) + } + if len(tree.names) > 3*burstSlack || len(tree.frames) > 2*burstSlack || len(tree.frameData) > 2*burstSlack { + t.Fatalf("after %d records the parse holds %d names and %d frames (%d bindings)", + n, len(tree.names), len(tree.frames), len(tree.frameData)) + } + if len(got) != n { + t.Fatalf("%d bursts", len(got)) + } + for i, r := range got { + c := r.FirstChild() + a := c.AttrAt(0) + ns := r.InScopeNamespaces() + if c.Name().Local != fmt.Sprintf("n%d", i) || a.Name().URI != fmt.Sprintf("urn:%d", i) || + ns[fmt.Sprintf("p%d", i)] != fmt.Sprintf("urn:%d", i) { + t.Fatalf("burst %d reads %v @%v %v", i, c.Name(), a.Name(), ns) + } + } +} diff --git a/xdm/parse.go b/xdm/parse.go index b74e40d0..7243c586 100644 --- a/xdm/parse.go +++ b/xdm/parse.go @@ -370,7 +370,7 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { if _, text := tok.(*xml.CharData); !text { run.flush(&spaces, stripSpaceAt) if bs != nil { - bs.trimLeaves(cur, depth, &spaces) + bs.trimLeaves(cur, depth, &spaces, &names) } } @@ -419,7 +419,7 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { return nil, err } if bs != nil && cur == tree.Root { - bs.marks = append(bs.marks[:0], tree.text.mark()) + bs.marks = append(bs.marks[:0], markOf(tree)) bs.v11 = dec.IsVersion11() } el, decls := buildElement(t, cur, &ns, &names) @@ -483,7 +483,7 @@ func parse(r io.Reader, whole string, opts ParseOptions) (*Tree, error) { cur = cur.Parent() depth-- if bs != nil { - if err := bs.end(closed, depth+1, &spaces); err != nil { + if err := bs.end(closed, depth+1, &spaces, &names); err != nil { return nil, err } } From 311a711b45bc5bdcf9bb556a768655d542a5cfba Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:44:45 -0500 Subject: [PATCH 24/45] docs: the burst error-order classes the differential fuzzer found --- docs/known-gaps.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/known-gaps.md b/docs/known-gaps.md index 316b19a2..fe9bdbe2 100644 --- a/docs/known-gaps.md +++ b/docs/known-gaps.md @@ -185,9 +185,14 @@ burst path; the commonest refusals are a body of `xsl:value-of` (840), - **The result is still built in memory**, so a large filtered copy grows with its output and stops at the result-node cap. -- **Error order can differ from tree mode.** A dynamic error in the body is - raised before a malformed tail later in the file would be; tree mode reads - the whole file first. `xsl:try` can see which code wins. +- **Error order can differ from tree mode.** Tree mode reads the whole file, + then evaluates the whole `select`, before the body runs; bursts do each per + record. So a dynamic error in the body is raised before a malformed tail, a + `MaxNodes`/`MaxBytes` refusal or an encoding error later in the file, and + before a predicate's error on a later record (`a/r[xs:integer(@v) gt 0]` + with `1 idiv 0` in the body: FOAR0001 in bursts, FORG0001 in the tree). + `xsl:try` can see which code wins. A differential fuzzer (1.28M generated + cases, 495k on the burst path) found no other difference. - **`xsl:break` still reads the rest of the file**, so that a malformed tail is an error as in tree mode. - **Only `FileResolver` streams**; another `DocumentResolver` builds the tree. From 8aa888246774158bfb3e1ca129472ab79100b45b Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:46:14 -0500 Subject: [PATCH 25/45] xdm: a large burst's tree takes over the parse's chunks deliver copied the whole parse tree while it was alive, so one large record peaked at twice the tree. The chunks past the record's first are moved into the burst's tree; truncate reaches its tables by index. --- xdm/build.go | 15 ++++++++++----- xdm/bursts.go | 18 +++++++++++++++++- xdm/bursts_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++ xdm/clone.go | 28 ++++++++++++++++++++++----- 4 files changed, 97 insertions(+), 11 deletions(-) diff --git a/xdm/build.go b/xdm/build.go index 7d725d46..642de56e 100644 --- a/xdm/build.go +++ b/xdm/build.go @@ -244,7 +244,9 @@ func CopyTypeEnvs(dst, src *Node) { } } -// truncate drops the records from index i on. +// truncate drops the records from index i on. It reaches t's tables by +// index, not through the records, which a burst's tree may have taken over +// (see cloneMoving). func (t *Tree) truncate(i uint32) { for len(t.open) > 0 && t.open[len(t.open)-1] >= i { t.open = t.open[:len(t.open)-1] @@ -254,18 +256,21 @@ func (t *Tree) truncate(i uint32) { if r.flags&fBase != 0 { delete(t.bases, j) } + k, off := chunkOf(j) if r.flags&fTyped != 0 { - *r.typ() = nodeTyping{} + t.typing[k][off] = nodeTyping{} } if r.flags&fManyAttrs != 0 { if t.source != nil { delete(t.source.attrCounts, j) } } - r.setOffset(0) delete(t.docURIs, j) - if t.source != nil { - delete(t.source.foreignPos, j) + if s := t.source; s != nil { + if k < len(s.offsets) && s.offsets[k] != nil { + s.offsets[k][off] = 0 + } + delete(s.foreignPos, j) } } if t.side != nil { diff --git a/xdm/bursts.go b/xdm/bursts.go index d6753d00..87275180 100644 --- a/xdm/bursts.go +++ b/xdm/bursts.go @@ -53,6 +53,9 @@ type burster struct { // the document node) had stored its attributes. Everything stored after // it belongs to content that is dropped. marks []burstMark + // moved is the first of the parse's chunks the last burst's tree took + // over, or 0. + moved int v11 bool } @@ -124,6 +127,12 @@ func (b *burster) trimLeaves(cur *Node, d int, spaces *spaceTable, names *nameCa // frames stored since p's spine mark; names is the parse's name cache. func (b *burster) discardLast(p *Node, spaces *spaceTable, names *nameCache) { p.RemoveLastChild() + if t := p.tree; b.moved > 0 { + // Those chunks are the burst's now; the parse allocates afresh. + clear(t.chunks[b.moved:]) + t.chunks = t.chunks[:b.moved] + b.moved = 0 + } m := b.marks[len(b.marks)-1] p.tree.text.rewind(m.text) rewindNames(p.tree, m.names, names) @@ -138,8 +147,15 @@ func (b *burster) discardLast(p *Node, spaces *spaceTable, names *nameCache) { // keep the prolog's comments and processing instructions, which every burst // would copy), which is exactly the burst's tree: it is // copied in bulk, sharing the names, frames and text the records point into. +// The chunks wholly past el's first are not copied but taken over, so that +// one large burst is not held twice; discardLast drops them from the parse. func (b *burster) deliver(el *Node) error { - m := cloneSubtree(el.tree.Root, xdmclone.Options{}) + t := el.tree + m := cloneMoving(t.Root, xdmclone.Options{}, el.self) + k, _ := chunkOf(el.self) + if last, _ := chunkOf(t.n - 1); last > k { + b.moved = k + 1 + } c := m(el) c.tree.XMLVersion = "1.0" if b.v11 { diff --git a/xdm/bursts_test.go b/xdm/bursts_test.go index 8f697df4..46bd9a75 100644 --- a/xdm/bursts_test.go +++ b/xdm/bursts_test.go @@ -234,3 +234,50 @@ func TestParseBurstsNamesStaySmall(t *testing.T) { } } } + +// A large burst's records are taken over by its tree, not copied while the +// parse still holds them, and stay intact as the parse goes on. +func TestParseBurstsLargeNotCopied(t *testing.T) { + var b strings.Builder + b.WriteString(`]>`) + for i := range 6 { + n := 3 + if i%2 == 0 { + n = 20000 + } + fmt.Fprintf(&b, ``, i) + for j := range n { + fmt.Fprintf(&b, `t`, i, j, j%3) + } + b.WriteString("") + } + b.WriteString("") + src := b.String() + opts := ParseOptions{AllowDOCTYPE: true} + path := []QName{{Local: "a"}, {Local: "r"}} + tree, err := ParseString(src, opts) + if err != nil { + t.Fatal(err) + } + var got []*Node + burst := allocated(func() { + err = ParseBursts(strings.NewReader(src), opts, path, func(r *Node) error { got = append(got, r); return nil }) + }) + if err != nil { + t.Fatal(err) + } + whole := allocated(func() { _, err = ParseString(src, opts) }) + if burst > whole*5/4 { + t.Fatalf("bursts allocated %d bytes, the whole tree %d", burst, whole) + } + i := 0 + for r := range tree.Root.FirstChild().Children() { + if g, w := dump(got[i])+ancestry(got[i]), dump(r)+ancestry(r); g != w { + t.Fatalf("burst %d differs\nwant\n%.2000s\ngot\n%.2000s", i, w, g) + } + if c := got[i].LastChild(); !c.AttrAt(0).IsID() || c.BaseURI() != r.LastChild().BaseURI() { + t.Fatalf("burst %d: the last child lost its ID type or base URI", i) + } + i++ + } +} diff --git a/xdm/clone.go b/xdm/clone.go index 784aad69..d50917c7 100644 --- a/xdm/clone.go +++ b/xdm/clone.go @@ -25,6 +25,15 @@ func init() { // point into are append-only, so the copy shares them, with capacities // clipped so that whatever the copy appends lands in storage of its own. func cloneSubtree(top *Node, o xdmclone.Options) func(*Node) *Node { + return cloneMoving(top, o, noIdx) +} + +// cloneMoving is cloneSubtree that, cloning a whole tree as it is, takes +// over the source's chunks that lie wholly past the one holding record from +// instead of copying them: their records become the copy's, in place. The +// source must not read them through its own records again, and must drop +// them (see burster.deliver). +func cloneMoving(top *Node, o xdmclone.Options, from uint32) func(*Node) *Node { if top.flags&fSide != 0 { return nil } @@ -128,9 +137,17 @@ func cloneSubtree(top *Node, o xdmclone.Options) func(*Node) *Node { total = newIdx[n] } last, _ := chunkOf(total - 1) + keep := last // the last chunk allocated rather than taken over + if from != noIdx && lo == 0 && newIdx == nil { + keep, _ = chunkOf(from) + keep = min(keep, last) + } c.chunks = make([][]Node, 0, last+1) - for len(c.chunks) <= last { - c.chunks = appendChunk(c.chunks, len(c.chunks), last) + for len(c.chunks) <= keep { + c.chunks = appendChunk(c.chunks, len(c.chunks), keep) + } + if keep < last { + c.chunks = append(c.chunks, t.chunks[keep+1:last+1]...) } c.n = total @@ -198,16 +215,17 @@ func cloneSubtree(top *Node, o xdmclone.Options) func(*Node) *Node { } } } + // Read before r is placed, which in a chunk taken over is over s. + ty, typed := s.typ(), s.flags&fTyped != 0 d := place(j, r) - from, typed := s.typ(), s.flags&fTyped != 0 if layer != nil { if lt, ltyped, ok := layer.typingAt(i); ok { - from, typed = lt, ltyped + ty, typed = lt, ltyped } } if typed { dt := d.ownTyping() - *dt = *from // the names are shared, not copied + *dt = *ty // the names are shared, not copied if o.Detached && dt.get().env != nil { v := *dt.names v.env = nil From 5a37a75a8f4c01e8f983720ca5dedcd8641ccd98 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 15:58:46 -0500 Subject: [PATCH 26/45] docs: test count 2,984 after the audit fixes --- CHANGELOG.md | 2 +- README.md | 4 ++-- docs/conformance-gaps.md | 2 +- docs/stats.md | 2 +- docs/testing.md | 2 +- docs/todo.md | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b0175a0..52e54d77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ is byte-identical; new API is additive. | Change | What it does | Commit | |---|---|---| | Streamed `xsl:source-document`, first stage | A streamable `xsl:for-each`/`xsl:iterate` over a child path reads one record at a time: 1 GB totalled at 27 MB RSS, not 3.7 GB, at about 2× CPU. Other shapes build the tree. | [`3f185fe`][3f185fe], [`27757b3`][27757b3] | -| `xdm.ParseBursts` | Parses a document handing over one selected element at a time in a tree of its own, with limits counted over the whole document. | [`b4b6052`][b4b6052], [`80f999f`][80f999f] | +| `xdm.ParseBursts` | Parses a document handing over one selected element at a time in a tree of its own, with limits counted over the whole document; memory stays at the spine plus one record. | [`b4b6052`][b4b6052], [`80f999f`][80f999f] | | `xslt.FileResolver.MaxNodes`; `MaxBytes` reaches the parser | Raising `MaxBytes` past 64 MB had no effect on `fn:doc`, the parser kept its own default. Both limits now apply to what the resolver parses. | [`80f999f`][80f999f] | | `xdmbuild.InPlaceDocument` | A builder rooted at a new document node, for content built straight into it. | [`302c932`][302c932] | diff --git a/README.md b/README.md index 4b1e03f7..075ce56e 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ and maintains it as a project of his own. | **XSD 1.1** | 99.98% instance (26,217 of 26,222); **99.97%** schema-validity (15,350 of 15,354); opt-in via `Version11` | | **RELAX NG** | 100.00% of James Clark's spectest (965 of 965 assertions); XML and compact syntax, from Go and from `go-xml validate -rng` | | **DTD** | content models, attribute defaults, enumerations, `ID`/`IDREF`; external subset, parameter entities across both subsets, conditional sections — via `dtd.Load` with a caller-supplied resolver, nothing fetched by default | -| **Tests** | 2,976 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | +| **Tests** | 2,984 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | | **Production schemas** | UBL 2.1, UN/CEFACT CII, Factur-X/ZUGFeRD, Peppol BIS 3.0 — 88 schemas load, instances validate clean | | **API** | 1.2; the exported surface is stable and additive over 1.1, and a breaking change means 2.0 with a new module path | @@ -1391,7 +1391,7 @@ back, is in [docs/testing.md](docs/testing.md). | method | what it catches | what it misses | |---|---|---| -| **Unit tests** (2,976 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** (2,984 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | | **Spec inventories** | features absent entirely | features present but behaving wrongly | | **Saxon differential** | subtle behavioural divergence on real stylesheets | constructs the corpora do not use | | **W3C QT3 suite** | systematic conformance across 22,054 XPath and 30,517 XQuery cases | XSLT (it is an XPath suite) | diff --git a/docs/conformance-gaps.md b/docs/conformance-gaps.md index 4d10d7f5..996ccd03 100644 --- a/docs/conformance-gaps.md +++ b/docs/conformance-gaps.md @@ -49,7 +49,7 @@ W3C disagreements: 0 + 0 + 0 + 1 + 8 + 23 + 30 + 31 + 0 = 93. Measured 2026-09-1 -The unit-test suite is 2,976 tests. +The unit-test suite is 2,984 tests. The last two rows are not W3C suites but real-world corpora — DocBook xslTNG's diff --git a/docs/stats.md b/docs/stats.md index 64e66c53..d084fcf7 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -23,7 +23,7 @@ claim is only as good as the command behind it. | figure | count | counted by | |---|---:|---| -| Unit tests | 2,976 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | +| Unit tests | 2,984 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Fuzz targets | 12 | `grep -rn '^func Fuzz' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Limit boundary tests | 14 | `grep -hc '^func Test' ./*/limits_boundary*_test.go \| awk '{n += $1} END {print n + 0}'` | diff --git a/docs/testing.md b/docs/testing.md index 86f81108..a2fd5525 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,7 +21,7 @@ let something through, and the column that matters is the last one. | layer | count | catches | misses | |---|---:|---|---| -| **Unit tests** | 2,976 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** | 2,984 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | | **Limit boundary tests** | 14 tests | an off-by-one or an overflow at the edge of a configurable limit | a limit nobody added to the inventory | | **Race detector** | same tests | shared state a single-goroutine run never reveals | a data race on a path no test walks | | **W3C conformance suites** | 152,241 cases | systematic divergence from the specification | what the suites do not ask about — see below | diff --git a/docs/todo.md b/docs/todo.md index d53e80a9..6f53d934 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -23,7 +23,7 @@ Current position: | XSLT 3.0 | 99.80% — 11,495 of 11,518 in scope (23 failing); 3 of those need more of the §19.8 streamability analysis | | RELAX NG | 100.00% — 965 of 965 | | Schemas wrongly refused | 7 — 6 on XSD 1.0, 1 on 1.1 | -| Tests | 2,976 `func Test` declarations, clean under `-race` | +| Tests | 2,984 `func Test` declarations, clean under `-race` | Every one of those failures, and why it is still open, is catalogued in [known-gaps.md](known-gaps.md). This file is the forward-looking half — what From c7952b0c02f72a4787282ece1197ed09845d15f2 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 16:42:05 -0500 Subject: [PATCH 27/45] xslt: keep no copy-to-original table when no accumulator is declared fn:copy-of, fn:snapshot and xsl:copy-of copy-accumulators=yes recorded every copied node against its original in rt.accumOrigin for the whole transform, pinning every copy and every streamed burst tree. The table is only read by accumulator-before/after, which cannot resolve a name when no package declares an accumulator (all packages file into Stylesheet.accumulators), so it is now nil then. Streamed copy-of(.) per record at 10 MB: 830 MB -> 2.8 MB peak heap. --- xslt/accumulator.go | 19 ++++++++++ xslt/copyorigin_test.go | 81 +++++++++++++++++++++++++++++++++++++++++ xslt/runtime.go | 7 +++- 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 xslt/copyorigin_test.go diff --git a/xslt/accumulator.go b/xslt/accumulator.go index 844850d0..b93b6e7e 100644 --- a/xslt/accumulator.go +++ b/xslt/accumulator.go @@ -714,6 +714,25 @@ func (rt *runtime) noteCopiedAccumulators(orig, copy *xdm.Node) { pair(orig, copy) } +// newAccumOrigin is the copy-to-original table of a runtime for s, or nil when +// nothing can ever read it. +// +// §18.2.2 makes an accumulator applicable to a copy made by fn:copy-of, +// fn:snapshot or xsl:copy-of with copy-accumulators="yes" "if and only if it +// is applicable to" the original's tree, and §18.2.10 gives the copy the +// original's values. Both presuppose an accumulator: a stylesheet none of whose +// packages declares one -- every package files its declarations in +// s.accumulators -- has no name accumulator-before or accumulator-after can +// resolve, so the table would only hold every copied node, and its original, +// alive until the transform ends. That was 830 MB of peak heap for a streamed +// 10 MB input taking copy-of(.) of each record. +func newAccumOrigin(s *Stylesheet) map[*xdm.Node]*xdm.Node { + if len(s.accumulators) == 0 { + return nil + } + return map[*xdm.Node]*xdm.Node{} +} + // accumulatorOrigin follows a node back to the node it was copied from, or // returns it unchanged. The chain is followed to its end so that a copy of a // copy still answers with the original's value. diff --git a/xslt/copyorigin_test.go b/xslt/copyorigin_test.go new file mode 100644 index 00000000..9e81342b --- /dev/null +++ b/xslt/copyorigin_test.go @@ -0,0 +1,81 @@ +package xslt + +import ( + "context" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// A copy is recorded against its original only when some package declares an +// accumulator that could read it back. +// +// §18.2.10 gives a node copied by fn:copy-of, fn:snapshot or xsl:copy-of with +// copy-accumulators="yes" its original's accumulator values, and the runtime +// keeps a copy-to-original table to answer them. The table holds both trees +// until the transform ends, so recording into it when no accumulator exists +// kept every copy-of(.) of a streamed record alive: 830 MB of peak heap for a +// 10 MB input. With an accumulator declared -- here only in a used package, +// which files it in the same table as the top-level package -- the copy must +// still answer as its original: x[2] counts 2 there, while its copy, walked +// as a tree of its own, would count 1. +func TestCopyOriginOnlyWithAccumulators(t *testing.T) { + plain := compileString(t, ``) + rt, err := newRuntime(plain, context.Background(), nil, TransformOptions{}) + if err != nil { + t.Fatal(err) + } + doc, err := xdm.ParseString(``, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 100; i++ { + noteCopy(rt, doc.Root, copyItem(doc.Root)) + noteCopy(rt, doc.Root, snapshotItem(doc.Root)) + } + if n := len(rt.accumOrigin); n != 0 { + t.Errorf("no accumulator is declared, yet %d copied nodes are held "+ + "against their originals", n) + } + + const base = ` + + + + + + + + ` + const top = ` + + + + + {p:after(copy-of($d//x[2]))} + + ` + tree, err := xdm.ParseString(top, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + s, err := Compile(tree.Root, CompileOptions{PackageResolver: fixedPackages{"urn:base": base}}) + if err != nil { + t.Fatalf("compile: %v", err) + } + res, err := s.Transform(context.Background(), nil, + TransformOptions{InitialTemplate: "initial-template", InitialTemplateURI: xdm.NSXSL}) + if err != nil { + t.Fatalf("transform: %v", err) + } + if got := res.String(); !strings.Contains(got, "2") { + t.Errorf("got %q, want 2: a copy answers accumulator-after "+ + "with its original's value", got) + } +} diff --git a/xslt/runtime.go b/xslt/runtime.go index 0f7552f0..07287806 100644 --- a/xslt/runtime.go +++ b/xslt/runtime.go @@ -217,6 +217,11 @@ type transformState struct { // an accumulator's value at the copy should be. It is a map on the // runtime rather than a field on the node so that copying leaves the // tree itself untouched. + // + // It is nil when no package of the stylesheet declares an accumulator: + // accumulator-before and -after then fail XTDE3340 before they could + // consult it, and every entry would pin a copy and its original for the + // rest of the transform. See newAccumOrigin. accumOrigin map[*xdm.Node]*xdm.Node // treeAccums records, per document root, which accumulators 18.2.2 makes @@ -765,7 +770,7 @@ func newRuntime(s *Stylesheet, ctx context.Context, root *xdm.Node, opts Transfo accumValues: map[accumCacheKey]*accumulatorValues{}, accumBuilding: map[accumCacheKey]*accumulatorValues{}, - accumOrigin: map[*xdm.Node]*xdm.Node{}, + accumOrigin: newAccumOrigin(s), treeAccums: map[*xdm.Node]*modeAccumulators{}, streamedTrees: map[*xdm.Node]bool{}, funcResults: map[callHash]xdm.Sequence{}, From 0920189aac70aa8ae0c8056d9fcc15c1d2b6f2e1 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 16:51:12 -0500 Subject: [PATCH 28/45] docs: the copy-to-original table fix and what it still retains --- CHANGELOG.md | 2 ++ docs/known-gaps.md | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52e54d77..1c176a25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ is byte-identical; new API is additive. | Built-in shallow-copy built a document result twice | In a mode with no template rules it builds in place: parse-100mb identity peak 1091 → 930 MB, CPU −25 to −40%. | [`302c932`][302c932] | | Built trees stored every whitespace-only text value | They share one value per string, as a parse does: an identity result is the size of its source again. | [`c60f8f3`][c60f8f3], [`09f0c5d`][09f0c5d] | | A deep copy duplicated the source's text | `xsl:copy-of` of 4,096+ records borrows the source blocks it fills by half or more: XMark identity result −28%, peak −19%. | [`546af03`][546af03], [`b4cd614`][b4cd614] | +| Every `copy-of` was kept until the transform ended | Each copied node was recorded against its original for accumulators, even with none declared: a 10 MB stream copying each record peaked at 830 MB. The table exists only when an accumulator does: 2.8 MB. Also in v2.1.1. | [`c7952b0`][c7952b0] | | Full record chunks were padded 6% by the malloc header | They are allocated two at a time past the 8th, 40,960 B exactly: parse-100mb retained 370 → 350.5 MB. | [`907c674`][907c674] | ## v2.1.1 — 2026-11-01 @@ -1867,3 +1868,4 @@ here so every entry in this file sits under a release. [546af03]: https://github.com/knroy/go-xml/commit/546af03 [b4cd614]: https://github.com/knroy/go-xml/commit/b4cd614 [907c674]: https://github.com/knroy/go-xml/commit/907c674 +[c7952b0]: https://github.com/knroy/go-xml/commit/c7952b0 diff --git a/docs/known-gaps.md b/docs/known-gaps.md index fe9bdbe2..cee26a9d 100644 --- a/docs/known-gaps.md +++ b/docs/known-gaps.md @@ -167,6 +167,12 @@ Measured on branches in 2026-11 and not adopted, each with the reason: - **Template-driven shallow-copy does not share source text.** Deep copies (`xsl:copy-of`, `xdm.Copy` of 4,096+ records) borrow the source's text blocks; a template-built copy adds each value as a string. +- **With any accumulator declared, every `fn:copy-of`/`fn:snapshot` copy is kept + until the transform ends**, with its original: §18.2.2 and §18.2.10 make the + copy answer with the original's accumulator values, and the copy-to-original + table is how it finds them. Weak references would not free them, since the + accumulator value cache holds every tree an accumulator was evaluated over. + With no accumulator in any package the table is not made. - **Holding millions of namespace nodes costs a sweep per collection.** Namespace nodes are held weakly since v2.2.0; while a tree's table is not empty, every GC cycle sweeps it. From b12ecbf6acafcc9767938528a8ca0ab7f68bb959 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:00:24 -0500 Subject: [PATCH 29/45] =?UTF-8?q?xslt:=20mark=20=C2=A719.8=20verdicts=20th?= =?UTF-8?q?at=20rest=20on=20an=20unverified=20declaration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The analysis takes several declarations at their word: a declared-streamable function by its category, a streamable attribute set, an accumulator named by accumulator-before/after, the rules of a streamable mode, and anything another module can override. Each declaration's own check stays silent when it meets an unmodelled construct, so a container could be known and grounded while resting on a declaration nobody verified. An unverified bit now travels beside known through every analyzer, set at each of those trust points unless a declTrust verifies the declaration on demand (memoised; in-progress counts as verified, per §19.8.5). Refusals read known only, so XTSE3430 results are unchanged (TestXSLT30Suite 11495/23). The burst gate now assesses the body with the module's declarations and requires known && !unverified. --- xslt/sourcedoc.go | 4 +- xslt/streamability.go | 31 ++++ xslt/streamburst.go | 21 ++- xslt/streamexprs.go | 3 + xslt/streamfunctions.go | 31 +++- xslt/streaminstructions.go | 136 +++++++++++---- xslt/streamtrust.go | 335 +++++++++++++++++++++++++++++++++++++ xslt/streamtrust_test.go | 194 +++++++++++++++++++++ 8 files changed, 711 insertions(+), 44 deletions(-) create mode 100644 xslt/streamtrust.go create mode 100644 xslt/streamtrust_test.go diff --git a/xslt/sourcedoc.go b/xslt/sourcedoc.go index 9d4ff56a..ea471e7c 100644 --- a/xslt/sourcedoc.go +++ b/xslt/sourcedoc.go @@ -71,7 +71,9 @@ func (c *compiler) compileSourceDocument(n *xdm.Node) (Instruction, error) { } var plan *burstPlan if isYes(n.AttrValue("streamable")) { - plan = planBursts(n) + // A module other than the principal one, or a principal that pulls + // in others, is not the whole stylesheet. + plan = planBursts(n, n.Root() != c.sheet.source) } outer := c.burst c.burst = plan diff --git a/xslt/streamability.go b/xslt/streamability.go index adf04269..33c4d899 100644 --- a/xslt/streamability.go +++ b/xslt/streamability.go @@ -41,6 +41,15 @@ type analyzer struct { // implementation rather than a fact about the stylesheet. known bool + // unverified is set where the verdict rests on a declaration whose own + // check did not positively pass, or that another module could override; + // trust verifies declarations on demand, and is nil on the refusal paths, + // which never read the bit. See streamtrust.go. The bit travels exactly + // as known does: every analyzer derived from this one starts from it and + // merges it back. + unverified bool + trust *declTrust + // funcs indexes the stylesheet's own xsl:function declarations, so that // a call on one can be assessed under §19.8.5 instead of abandoning the // enclosing construct. Nil when no declarations were collected, which @@ -349,6 +358,8 @@ func (a *analyzer) higherOrderOperand(e xpath.Expr, u usage) operand { ctxPosture: a.ctxPosture, ctxAllowsChildren: a.ctxAllowsChildren, known: a.known, + unverified: a.unverified, + trust: a.trust, funcs: a.funcs, streamingParam: a.streamingParam, paramCategory: a.paramCategory, @@ -366,6 +377,7 @@ func (a *analyzer) higherOrderOperand(e xpath.Expr, u usage) operand { } p := inner.expr(e) a.known = a.known && inner.known + a.unverified = a.unverified || inner.unverified return operand{ props: p, usage: u, @@ -479,6 +491,8 @@ func (a *analyzer) filter(x *xpath.FilterExpr) props { ctxPosture: cur.posture, ctxAllowsChildren: a.allowsChildren(x.Base), known: a.known, + unverified: a.unverified, + trust: a.trust, funcs: a.funcs, streamingParam: a.streamingParam, paramCategory: a.paramCategory, @@ -496,6 +510,7 @@ func (a *analyzer) filter(x *xpath.FilterExpr) props { } pp := inner.expr(p) a.known = a.known && inner.known + a.unverified = a.unverified || inner.unverified if pp.sweep != sweepMotionless { return roamingFreeRanging } @@ -538,6 +553,8 @@ func (a *analyzer) step(s *xpath.Step, ctx posture) props { ctxPosture: base.posture, ctxAllowsChildren: stepAllowsChildren(s), known: a.known, + unverified: a.unverified, + trust: a.trust, funcs: a.funcs, streamingParam: a.streamingParam, paramCategory: a.paramCategory, @@ -555,6 +572,7 @@ func (a *analyzer) step(s *xpath.Step, ctx posture) props { } pp := inner.expr(p) a.known = a.known && inner.known + a.unverified = a.unverified || inner.unverified if pp.sweep != sweepMotionless { return roamingFreeRanging } @@ -626,6 +644,8 @@ func (a *analyzer) path(x *xpath.PathExpr) props { ctxPosture: cur.posture, ctxAllowsChildren: curAllowsChildren, known: a.known, + unverified: a.unverified, + trust: a.trust, funcs: a.funcs, streamingParam: a.streamingParam, paramCategory: a.paramCategory, @@ -644,6 +664,7 @@ func (a *analyzer) path(x *xpath.PathExpr) props { next = inner.expr(e) curAllowsChildren = inner.allowsChildren(e) a.known = a.known && inner.known + a.unverified = a.unverified || inner.unverified } if !next.streamable() { prefix := x.Steps[:i+1] @@ -760,6 +781,7 @@ func (a *analyzer) isScanningStep(e xpath.Expr) bool { ctxPosture: postureStriding, ctxAllowsChildren: stepAllowsChildren(s), known: true, + trust: a.trust, funcs: a.funcs, streamingParam: a.streamingParam, paramCategory: a.paramCategory, @@ -776,6 +798,7 @@ func (a *analyzer) isScanningStep(e xpath.Expr) bool { decl: a.decl, } sw := inner.expr(p).sweep + a.unverified = a.unverified || inner.unverified if !inner.known { // The predicate holds something this analysis does not // model, so "not a scanning expression" is a statement @@ -993,6 +1016,9 @@ func (a *analyzer) funcCall(x *xpath.FuncCall) props { if arg := a.expr(x.Args[0]); arg.sweep != sweepMotionless { return props{postureGrounded, sweepFreeRanging} } + if !a.trust.accumulatorVerified(a.decl, x.Args[0]) { + a.unverified = true + } sw, ok := accumulatorAfterSweep(a.accumAfter, a.ctxPosture, a.ctxAllowsChildren) if !ok { return a.unknown() @@ -1011,6 +1037,11 @@ func (a *analyzer) funcCall(x *xpath.FuncCall) props { if arg := a.expr(x.Args[0]); arg.sweep != sweepMotionless { return roamingFreeRanging } + // Grounded because the accumulator is streamable: the trust that + // rests on its declaration. + if !a.trust.accumulatorVerified(a.decl, x.Args[0]) { + a.unverified = true + } return groundedMotionless } diff --git a/xslt/streamburst.go b/xslt/streamburst.go index 6f3a4abc..55d968c4 100644 --- a/xslt/streamburst.go +++ b/xslt/streamburst.go @@ -77,18 +77,19 @@ type burstResolver interface { } // planBursts decides, from the stylesheet source, whether the streamable -// xsl:source-document sd can run in bursts. -func planBursts(sd *xdm.Node) *burstPlan { +// xsl:source-document sd can run in bursts. shared says sd's module is not +// the whole stylesheet (streamtrust.go). +func planBursts(sd *xdm.Node, shared bool) *burstPlan { for _, a := range []string{"validation", "type", "use-accumulators"} { if sd.Attr("", a) != nil { return nil } } // The §19.8 verdict first: the body must be guaranteed-streamable, from - // constructs the analysis fully models. Everything below narrows that to - // what bursts can execute. - if p, known := analyzeSequenceConstructor(sd, postureStriding, nil, nil); !known || - !p.streamable() || p.posture != postureGrounded { + // constructs the analysis fully models, resting on no declaration that + // was not itself verified. Everything below narrows that to what bursts + // can execute. + if !burstVerdict(sd, shared) { return nil } el := sd @@ -118,6 +119,14 @@ func planBursts(sd *xdm.Node) *burstPlan { return &burstPlan{path: path, target: target, filtered: filtered} } +// burstVerdict reports whether the §19.8 verdict on sd is one execution may +// trust: known, unverified clear, and grounded. +func burstVerdict(sd *xdm.Node, shared bool) bool { + t := newDeclTrust(documentElementOf(sd), shared) + p, known, unverified := sequenceConstructorVerdict(sd, postureStriding, t.sets, t.funcs, t) + return known && !unverified && p.streamable() && p.posture == postureGrounded +} + // onlyElementChild is el's one element child, when el has exactly one and // nothing else but white space. func onlyElementChild(el *xdm.Node) *xdm.Node { diff --git a/xslt/streamexprs.go b/xslt/streamexprs.go index 038ae7ce..7dc5890b 100644 --- a/xslt/streamexprs.go +++ b/xslt/streamexprs.go @@ -296,6 +296,7 @@ func (a *analyzer) letExpr(x *xpath.LetExpr) props { inner.operandOf(ret, usageTransmission), }, false) a.known = a.known && inner.known + a.unverified = a.unverified || inner.unverified return p } @@ -315,6 +316,7 @@ func (a *analyzer) forExpr(x *xpath.ForExpr) props { inner.higherOrderOperand(ret, usageTransmission), }, false) a.known = a.known && inner.known + a.unverified = a.unverified || inner.unverified return p } @@ -381,6 +383,7 @@ func (a *analyzer) quantifiedExpr(x *xpath.QuantifiedExpr) props { inner := a.withVar(x.Bindings[0].Var, seq.props) op := inner.higherOrderOperand(test, usageInspection) a.known = a.known && inner.known + a.unverified = a.unverified || inner.unverified p := combine([]operand{seq, op}, false) if !p.streamable() { return roamingFreeRanging diff --git a/xslt/streamfunctions.go b/xslt/streamfunctions.go index da5df447..6cdd0166 100644 --- a/xslt/streamfunctions.go +++ b/xslt/streamfunctions.go @@ -540,6 +540,13 @@ func checkStreamableFunctions(root *xdm.Node, funcs map[funcKey]*streamFunc) err // rules for sequence constructors, which are not implemented, so it reports // "not known" and no error is raised for it. func analyzeFunctionBody(f *streamFunc, funcs map[funcKey]*streamFunc) (props, bool) { + p, known, _ := functionBodyVerdict(f, funcs, nil) + return p, known +} + +// functionBodyVerdict is analyzeFunctionBody plus the unverified bit, for +// declTrust.funcVerified. +func functionBodyVerdict(f *streamFunc, funcs map[funcKey]*streamFunc, trust *declTrust) (props, bool, bool) { var instr *xdm.Node for _, c := range f.body.ChildElements() { if isXSL(c, "param") { @@ -547,25 +554,25 @@ func analyzeFunctionBody(f *streamFunc, funcs map[funcKey]*streamFunc) (props, b } if instr != nil { // More than one instruction in the body. - return roamingFreeRanging, false + return roamingFreeRanging, false, false } instr = c } if instr == nil { // An empty body returns the empty sequence: grounded and motionless. - return groundedMotionless, true + return groundedMotionless, true, false } if !isXSL(instr, "sequence") && !isXSL(instr, "value-of") { - return roamingFreeRanging, false + return roamingFreeRanging, false, false } sel := instr.Attr("", "select") if sel == nil { - return roamingFreeRanging, false + return roamingFreeRanging, false, false } ns := newNSResolver(instr, xpathDefaultNamespace(instr)) expr, err := xpath.ParseVersion(sel.Value(), ns, xpathVersionAt(instr)) if err != nil { - return roamingFreeRanging, false + return roamingFreeRanging, false, false } // A recursive body needs no iteration to reach its fixed point, because @@ -597,6 +604,7 @@ func analyzeFunctionBody(f *streamFunc, funcs map[funcKey]*streamFunc) (props, b known: true, funcs: funcs, decl: instr, + trust: trust, } // The streaming parameter is the first parameter, and only when the // function is declared-streamable and that parameter's declared type @@ -611,7 +619,7 @@ func analyzeFunctionBody(f *streamFunc, funcs map[funcKey]*streamFunc) (props, b } p := a.expr(expr) if !a.known { - return roamingFreeRanging, false + return roamingFreeRanging, false, false } // xsl:value-of builds a text node from the atomized value, so its result @@ -638,7 +646,7 @@ func analyzeFunctionBody(f *streamFunc, funcs map[funcKey]*streamFunc) (props, b // children, what turns motionless into consuming. res := typeAdjust(p, f.asType, bodyAllowsChildren(instr, expr)) - return res, true + return res, true, a.unverified } // satisfiesCategory reports whether a function result meets the body rule of @@ -784,6 +792,8 @@ func (a *analyzer) simpleMap(x *xpath.SimpleMap) props { ctxPosture: left.posture, ctxAllowsChildren: a.allowsChildren(x.Left), known: a.known, + unverified: a.unverified, + trust: a.trust, funcs: a.funcs, streamingParam: a.streamingParam, paramCategory: a.paramCategory, @@ -807,6 +817,7 @@ func (a *analyzer) simpleMap(x *xpath.SimpleMap) props { } p := inner.expr(x.Right) a.known = a.known && inner.known + a.unverified = a.unverified || inner.unverified return p } @@ -834,6 +845,12 @@ func (a *analyzer) callProps(x *xpath.FuncCall) (props, bool) { if f.arity() == 0 { return groundedMotionless, true } + // The call is assessed from the declaration -- its category, or its + // parameter types -- so the verdict holds only once the declaration is + // verified and cannot be overridden from another module. + if f.category.declaredStreamable() && !a.trust.funcVerified(f) || !a.trust.local() { + a.unverified = true + } switch f.category { case catUnclassified: diff --git a/xslt/streaminstructions.go b/xslt/streaminstructions.go index 086d4432..b9fb8d90 100644 --- a/xslt/streaminstructions.go +++ b/xslt/streaminstructions.go @@ -45,6 +45,10 @@ type instrAnalyzer struct { // instruction operand is an XPath expression. known bool + // unverified and trust are analyzer's, merged the same way. + unverified bool + trust *declTrust + // attrSets resolves an attribute-set name to its declarations, for the // use-attribute-sets operand of §19.8.6. Nil when the enclosing // stylesheet was not scanned, in which case any use-attribute-sets @@ -124,17 +128,30 @@ func analyzeSequenceConstructor( el *xdm.Node, ctx posture, attrSets map[xdm.QName][]*xdm.Node, funcs map[funcKey]*streamFunc, ) (props, bool) { + p, known, _ := sequenceConstructorVerdict(el, ctx, attrSets, funcs, nil) + return p, known +} + +// sequenceConstructorVerdict is analyzeSequenceConstructor for a caller that +// will trust the verdict: it also reports whether the verdict rests on a +// declaration that trust could not verify (streamtrust.go). A nil trust +// verifies nothing. +func sequenceConstructorVerdict( + el *xdm.Node, ctx posture, attrSets map[xdm.QName][]*xdm.Node, + funcs map[funcKey]*streamFunc, trust *declTrust, +) (props, bool, bool) { a := &instrAnalyzer{ ctxPosture: ctx, ctxAllowsChildren: true, known: true, attrSets: attrSets, funcs: funcs, + trust: trust, // §19.8.9.4 for a container nested inside an xsl:for-each-group: see // enclosingGroupSelect. } var outerStreamed bool - a.currentGroup, outerStreamed = enclosingGroupSelect(el, attrSets, funcs) + a.currentGroup, outerStreamed, a.unverified = enclosingGroupSelect(el, attrSets, funcs, trust) a.groupInScope = a.currentGroup != props{} // A nested container whose enclosing group is not grounded reads streamed // nodes a second time. §19.8.9.4 calls that roaming, but this walk never @@ -144,7 +161,7 @@ func analyzeSequenceConstructor( // on the path itself, not by this clause. a.groupOutOfReach = !a.groupInScope && !outerStreamed && hasForEachGroupAncestor(el) p := a.body(el) - return p, a.known + return p, a.known, a.unverified } // enclosingGroupSelect gives the properties §19.8.9.4 assigns to a call on @@ -170,16 +187,20 @@ func analyzeSequenceConstructor( // withhold: the first is si-group-052, whose refusal §19.8.9.4 derives from // the group being streamed, and which is otherwise silenced by // groupOutOfReach. +// +// The third return value is the select's unverified bit, which the grounded +// answer rests on. func enclosingGroupSelect( el *xdm.Node, attrSets map[xdm.QName][]*xdm.Node, funcs map[funcKey]*streamFunc, -) (props, bool) { + trust *declTrust, +) (props, bool, bool) { for n := el.Parent(); n != nil; n = n.Parent() { if !isXSL(n, "for-each-group") { continue } at := n.Attr("", "select") if at == nil { - return props{}, false + return props{}, false, false } outer := &instrAnalyzer{ ctxPosture: postureStriding, @@ -187,17 +208,18 @@ func enclosingGroupSelect( known: true, attrSets: attrSets, funcs: funcs, + trust: trust, } sel := outer.exprPropsIn(at.Value(), n, postureStriding, true) if !outer.known { - return props{}, false + return props{}, false, false } if sel.posture != postureGrounded { - return props{}, true + return props{}, true, false } - return groundedMotionless, false + return groundedMotionless, false, outer.unverified } - return props{}, false + return props{}, false, false } // hasForEachGroupAncestor reports whether el is nested inside an @@ -224,6 +246,8 @@ func (a *instrAnalyzer) sub(ctx posture, allowsChildren bool) *instrAnalyzer { ctxPosture: ctx, ctxAllowsChildren: allowsChildren, known: a.known, + unverified: a.unverified, + trust: a.trust, attrSets: a.attrSets, currentGroup: a.currentGroup, groupInScope: a.groupInScope, @@ -281,6 +305,7 @@ func (a *instrAnalyzer) subFocus(ctx posture, allowsChildren bool) *instrAnalyze func (a *instrAnalyzer) mergeFrom(b *instrAnalyzer) { a.known = a.known && b.known + a.unverified = a.unverified || b.unverified } // exprProps assesses an XPath expression in this analyzer's context posture, @@ -315,6 +340,8 @@ func (a *instrAnalyzer) exprOperandIn(src string, el *xdm.Node, ctx posture, all ctxPosture: ctx, ctxAllowsChildren: allowsChildren, known: a.known, + unverified: a.unverified, + trust: a.trust, decl: el, currentGroup: a.currentGroup, groupInScope: a.groupInScope, @@ -330,6 +357,7 @@ func (a *instrAnalyzer) exprOperandIn(src string, el *xdm.Node, ctx posture, all p := an.expr(expr) kids := an.allowsChildren(expr) a.known = a.known && an.known + a.unverified = a.unverified || an.unverified return p, kids } @@ -704,6 +732,12 @@ func (a *instrAnalyzer) instruction(el *xdm.Node) props { // §19.8.4.4, which §19.8.4.29 makes the rule for xsl:next-match too: // an implicit context item operand (.) with usage absorption, plus // the xsl:with-param children. + // + // A streamed context item is handed to rules of the current mode, + // which are trusted to be streamable (streamtrust.go). + if a.ctxPosture != postureGrounded && !a.trust.enclosingRuleModesVerified(el) { + a.unverified = true + } ops := []operand{{ props: props{a.ctxPosture, sweepMotionless}, usage: usageAbsorption, @@ -850,6 +884,11 @@ func (a *instrAnalyzer) instruction(el *xdm.Node) props { // without them there is no verdict. return a.unknown() } + // The usages come from the target's declarations, which another + // module could override by import precedence. + if !a.trust.local() { + a.unverified = true + } var ci *xdm.Node for _, c := range target.ChildElements() { if isXSL(c, "context-item") { @@ -1003,8 +1042,13 @@ func (a *instrAnalyzer) useAttributeSetOperands(el *xdm.Node) []operand { ops = append(ops, operand{props: roamingFreeRanging, usage: usageAbsorption, allowsChildren: true}) continue } + p := a.declaredAttributeSetProps(decls) + if p == groundedMotionless && !a.trust.attrSetVerified(n, decls) { + // Motionless on the declaration's word (streamtrust.go). + a.unverified = true + } ops = append(ops, operand{ - props: a.declaredAttributeSetProps(decls), + props: p, usage: usageAbsorption, allowsChildren: true, }) @@ -1623,6 +1667,12 @@ func (a *instrAnalyzer) applyTemplates(el *xdm.Node) props { return roamingFreeRanging } + // The streamed selection is handed to the rules of the mode, which are + // trusted to be streamable (streamtrust.go). + if !a.appliedModeVerified(el) { + a.unverified = true + } + // Clause 5: the general rules, with the select absorbing. ops := []operand{{props: sel, usage: usageAbsorption, allowsChildren: selKids}} ops = append(ops, a.withParamOperands(el, nil)...) @@ -1681,33 +1731,16 @@ const ( // its absence is decisive. As soon as another module is pulled in, the // declaration may be there, and the answer is modeUnresolved. func (a *instrAnalyzer) modeStreamable(el *xdm.Node) modeVerdict { - raw := strings.TrimSpace(el.AttrValue("mode")) - if raw == "#current" { + if strings.TrimSpace(el.AttrValue("mode")) == "#current" { return modeStreamableYes } root := documentElementOf(el) if root == nil { return modeUnresolved } - // §3.7.2: an omitted mode attribute, or "#default", takes the mode from - // the [xsl:]default-mode of the innermost ancestor that has one, and only - // falls back to the unnamed mode when there is none. sf-current-100 - // declares default-mode="m" on xsl:stylesheet and relies on a bare - // xsl:apply-templates reaching the streamable mode m. - at, on := el, el - if raw == "" || raw == "#default" { - raw, on = defaultModeFor(el) - } - if on != nil { - at = on - } - want := xdm.QName{} - if raw != "" && raw != "#default" && raw != "#unnamed" { - n, err := resolveQNameAttr(at, raw) - if err != nil { - return modeUnresolved - } - want = n + want, ok := appliedModeName(el) + if !ok { + return modeUnresolved } declared := false streamable := false @@ -1751,6 +1784,46 @@ func (a *instrAnalyzer) modeStreamable(el *xdm.Node) modeVerdict { return modeNotStreamable } +// appliedModeName resolves the mode an xsl:apply-templates names other than +// "#current", and reports false when its prefix does not resolve. +// +// §3.7.2: an omitted mode attribute, or "#default", takes the mode from the +// [xsl:]default-mode of the innermost ancestor that has one, and only falls +// back to the unnamed mode when there is none. sf-current-100 declares +// default-mode="m" on xsl:stylesheet and relies on a bare xsl:apply-templates +// reaching the streamable mode m. +func appliedModeName(el *xdm.Node) (xdm.QName, bool) { + raw := strings.TrimSpace(el.AttrValue("mode")) + at, on := el, el + if raw == "" || raw == "#default" { + raw, on = defaultModeFor(el) + } + if on != nil { + at = on + } + if raw == "" || raw == "#default" || raw == "#unnamed" { + return xdm.QName{}, true + } + n, err := resolveQNameAttr(at, raw) + if err != nil { + return xdm.QName{}, false + } + return n, true +} + +// appliedModeVerified reports whether the rules an xsl:apply-templates +// dispatches a streamed selection to are verified streamable. +func (a *instrAnalyzer) appliedModeVerified(el *xdm.Node) bool { + if strings.TrimSpace(el.AttrValue("mode")) == "#current" { + return a.trust.enclosingRuleModesVerified(el) + } + want, ok := appliedModeName(el) + if !ok { + return false + } + return a.trust.modeVerified(documentElementOf(el), xdm.QName{URI: want.URI, Local: want.Local}.Clark()) +} + // documentElementOf returns the outermost element containing el. func documentElementOf(el *xdm.Node) *xdm.Node { top := el @@ -1979,6 +2052,9 @@ func (a *instrAnalyzer) forEachGroup(el *xdm.Node) props { } else if free && !patternPredicateOnlySyntacticallyNumeric(p.Value(), el) { return roamingFreeRanging } + if !patternVerified(p.Value(), el) { + a.unverified = true + } } if sel.posture == postureGrounded { diff --git a/xslt/streamtrust.go b/xslt/streamtrust.go new file mode 100644 index 00000000..4ffef4ed --- /dev/null +++ b/xslt/streamtrust.go @@ -0,0 +1,335 @@ +package xslt + +// Whether a §19.8 verdict may be trusted by the execution side. +// +// The analysis gives every construct a posture and sweep, and clears known +// where it meets something it does not model. That is enough for XTSE3430, +// which is raised only on a known verdict. It is not enough to drive +// execution, because several rules take a DECLARATION at its word: a call on a +// declared-streamable stylesheet function is assessed by its category +// (§19.8.5), a use-attribute-sets reference by the set's streamable attribute +// (§19.8.6), a call on fn:accumulator-before or -after as grounded (§19.8.9.1, +// §19.8.9.2), and an xsl:apply-templates into a streamable mode by the mode's +// declaration alone (§19.8.4.5). Each declaration has its own check, and each +// of those checks stays silent when it meets something unmodelled -- so a +// container can be known and grounded while resting on a declaration nobody +// verified. The m10 soundness lane found such stylesheets compile with a +// grounded verdict and read data a streaming evaluation would already have +// passed. +// +// So the analysis carries a second bit beside known: unverified, set where a +// verdict rests on a declaration whose own check did not positively pass, or +// on one another module could override. Refusals never read it, so XTSE3430 +// results cannot change; the burst gate (streamburst.go) requires it clear. +// +// The check of each declaration is made only on demand, through a declTrust +// the trusting caller creates. Without one -- every refusal path -- each trust +// point simply sets the bit, which nobody reads there, and costs nothing. + +import ( + "strings" + + "github.com/knroy/go-xml/v2/xdm" + "github.com/knroy/go-xml/v2/xpath" +) + +// declTrust memoises, for one module, whether each declaration a verdict can +// rest on passed its own check. +type declTrust struct { + // shared says the module is not the whole stylesheet: it includes, + // imports or uses other modules, or is itself one of them. Another + // module can then add template rules to a mode, or override a function, + // named template, attribute set or accumulator by import precedence, so + // no declaration seen here can be trusted. + shared bool + sets map[xdm.QName][]*xdm.Node + funcs map[funcKey]*streamFunc + + state map[any]trustState + // depth, chain, cyclic and failed implement the circularity rule below. + depth int + chain []any + cyclic, failed bool +} + +type trustState int8 + +const ( + trustUnseen trustState = iota + trustInProgress + trustVerified + trustUnverified +) + +func newDeclTrust(root *xdm.Node, shared bool) *declTrust { + return &declTrust{ + shared: shared || reachesOtherModules(root), + sets: attributeSetDeclarations(root), + funcs: collectStreamFuncs(root), + state: map[any]trustState{}, + } +} + +// reachesOtherModules reports whether a module pulls in another. +func reachesOtherModules(root *xdm.Node) bool { + found := false + walkElements(root, func(el *xdm.Node) bool { + if isXSL(el, "import") || isXSL(el, "include") || isXSL(el, "use-package") { + found = true + } + return !found + }) + return found +} + +// local reports whether declarations seen in this module are the ones in +// force: false without a declTrust, or in a module another can override. +func (t *declTrust) local() bool { return t != nil && !t.shared } + +// verified runs check for the declaration key once, and reports whether it +// passed. +// +// A declaration whose check is still in progress counts as verified: that is +// §19.8.5's own circularity break, where a recursive call is assessed by the +// callee's declared category without consulting its body. A result reached +// under such an assumption is provisional, so when a cycle was met and any +// check in the same outermost run failed, every result of that run is +// demoted -- an over-approximation, in the safe direction, of the greatest +// fixed point. +func (t *declTrust) verified(key any, check func() bool) bool { + if t == nil || t.shared { + return false + } + switch t.state[key] { + case trustInProgress: + t.cyclic = true + return true + case trustVerified: + return true + case trustUnverified: + return false + } + t.state[key] = trustInProgress + t.depth++ + ok := check() + t.depth-- + if ok { + t.state[key] = trustVerified + t.chain = append(t.chain, key) + } else { + t.state[key] = trustUnverified + t.failed = true + } + if t.depth == 0 { + if t.cyclic && t.failed { + for _, k := range t.chain { + t.state[k] = trustUnverified + } + } + t.chain, t.cyclic, t.failed = nil, false, false + } + return t.state[key] == trustVerified +} + +// funcVerified: a declared-streamable function's body meets its category +// (§19.8.5), fully modelled and resting on nothing unverified. +func (t *declTrust) funcVerified(f *streamFunc) bool { + return t.verified(f.body, func() bool { + if f.category != catAbsorbing && f.arity() > 0 && typePermitsSeveralNodes(f.params[0]) { + return false + } + p, known, unverified := functionBodyVerdict(f, t.funcs, t) + return known && !unverified && satisfiesCategory(f, p) + }) +} + +// attrSetKey keys an attribute set in declTrust.state. +type attrSetKey struct{ name xdm.QName } + +// attrSetVerified: every declaration of a streamable attribute set is +// motionless (§19.8.6), fully modelled and resting on nothing unverified. +func (t *declTrust) attrSetVerified(name xdm.QName, decls []*xdm.Node) bool { + return t.verified(attrSetKey{name}, func() bool { + a := &instrAnalyzer{ctxPosture: postureStriding, ctxAllowsChildren: true, + known: true, attrSets: t.sets, funcs: t.funcs, trust: t} + p := a.attributeSetProps(decls) + return a.known && !a.unverified && p.sweep == sweepMotionless + }) +} + +// accumulatorVerified: the accumulator a call on fn:accumulator-before or +// -after names is declared streamable in this module and meets every +// condition of §18.2.9, each fully modelled. +func (t *declTrust) accumulatorVerified(at *xdm.Node, nameArg xpath.Expr) bool { + if t == nil || at == nil { + return false + } + lit, ok := nameArg.(*xpath.Literal) + if !ok || lit.Val == nil { + return false + } + want, err := resolveQNameAttr(at, strings.TrimSpace(lit.Val.String())) + if err != nil { + return false + } + var acc *xdm.Node + walkElements(documentElementOf(at), func(d *xdm.Node) bool { + if acc == nil && isXSL(d, "accumulator") { + if n, e := resolveQNameAttr(d, d.AttrValue("name")); e == nil && n == want { + acc = d + } + return false + } + return acc == nil + }) + if acc == nil || !isYes(acc.AttrValue("streamable")) { + return false + } + return t.verified(acc, func() bool { return t.accumulatorOK(acc) }) +} + +func (t *declTrust) accumulatorOK(acc *xdm.Node) bool { + // Conditions 4 and 5 of §18.2.9 on an expression: grounded and + // motionless, the posture forgiven where the declared type atomizes + // (checkOneAccumulator's reading of §18.2.1). + exprOK := func(src string, at *xdm.Node, kids, atomic bool) bool { + ns := newNSResolver(at, xpathDefaultNamespace(at)) + e, err := xpath.ParseVersion(src, ns, xpathVersionAt(at)) + if err != nil { + return false + } + a := &analyzer{ctxPosture: postureStriding, ctxAllowsChildren: kids, + known: true, decl: at, funcs: t.funcs, trust: t} + p := a.expr(e) + return a.known && !a.unverified && p.sweep == sweepMotionless && + (p.posture == postureGrounded || atomic) + } + atomic := atomicSequenceType(acc.AttrValue("as")) + if a := acc.Attr("", "initial-value"); a != nil && !exprOK(a.Value(), acc, true, false) { + return false + } + for _, rule := range acc.ChildElements() { + if !isXSL(rule, "accumulator-rule") { + continue + } + if !patternVerified(rule.AttrValue("match"), rule) { + return false + } + kids := matchedNodeAllowsChildren(rule.AttrValue("match"), rule) + if a := rule.Attr("", "select"); a != nil { + if !exprOK(a.Value(), rule, kids, atomic) { + return false + } + continue + } + // The contained sequence constructor, which the refusal check + // leaves alone; here it is simply assessed. + p, known, unverified := sequenceConstructorVerdict(rule, postureStriding, t.sets, t.funcs, t) + if !known || unverified || p.sweep != sweepMotionless || + (p.posture != postureGrounded && !atomic) { + return false + } + } + return true +} + +// patternVerified: a pattern is motionless (§19.8.10), fully modelled, and +// calls no accumulator function -- the pattern classifier assesses those +// without a declTrust, so a call there is never taken on trust. +func patternVerified(src string, at *xdm.Node) bool { + if src == "" || strings.Contains(src, "accumulator-") { + return false + } + free, known := patternIsFreeRanging(src, at) + return known && !free +} + +// modeKey keys a mode in declTrust.state. +type modeKey struct{ clark string } + +// modeVerified: every template rule of a streamable mode, mode="#all" ones +// included, meets §6.6.4 -- a motionless pattern, motionless parameter +// defaults, a grounded body -- each fully modelled and resting on nothing +// unverified. A rule withheld from refusal (current-group(), say) has a body +// that is not known and grounded, so it fails here. +func (t *declTrust) modeVerified(root *xdm.Node, clark string) bool { + return t.verified(modeKey{clark}, func() bool { + ok := true + walkElements(root, func(el *xdm.Node) bool { + if !ok || !isXSL(el, "template") || el.Attr("", "match") == nil { + return ok + } + if !strings.Contains(" "+el.AttrValue("mode")+" ", " #all ") && + !containsString(templateModeNames(el), clark) { + return true + } + ok = t.ruleOK(el) + return ok + }) + return ok + }) +} + +func (t *declTrust) ruleOK(rule *xdm.Node) bool { + if !patternVerified(rule.AttrValue("match"), rule) { + return false + } + for _, c := range rule.ChildElements() { + if !isXSL(c, "param") { + continue + } + a := &instrAnalyzer{ctxPosture: postureStriding, ctxAllowsChildren: true, + known: true, attrSets: t.sets, funcs: t.funcs, trust: t} + var p props + if s := c.Attr("", "select"); s != nil { + p = a.exprProps(s.Value(), c) + } else { + p = a.body(c) + } + if !a.known || a.unverified || p.sweep != sweepMotionless { + return false + } + } + p, known, unverified := sequenceConstructorVerdict(rule, postureStriding, t.sets, t.funcs, t) + return known && !unverified && p.streamable() && p.posture == postureGrounded +} + +// enclosingRuleModesVerified is the trust an xsl:next-match, xsl:apply-imports +// or mode="#current" places in the rules of the mode the enclosing template +// rule was selected in. Every mode the rule belongs to must be streamable and +// verified: the current mode is whichever one selected it. +// +// §19.8.4.5 treats mode="#current" as streamable because a rule reached with +// a streamed context item must have been selected in a streamable mode. That +// does not hold inside an xsl:source-document within a rule of a +// non-streamable mode, which is why the modes are checked here rather than +// assumed. +func (t *declTrust) enclosingRuleModesVerified(el *xdm.Node) bool { + for n := el.Parent(); n != nil && n.Kind() == xdm.KindElement; n = n.Parent() { + if !isXSL(n, "template") { + continue + } + if n.Attr("", "match") == nil || strings.Contains(" "+n.AttrValue("mode")+" ", " #all ") { + return false + } + root := documentElementOf(n) + streamable := streamableModeNames(root) + modes := templateModeNames(n) + for _, m := range modes { + if !streamable[m] || !t.modeVerified(root, m) { + return false + } + } + return len(modes) > 0 + } + return false +} + +func containsString(ss []string, s string) bool { + for _, x := range ss { + if x == s { + return true + } + } + return false +} diff --git a/xslt/streamtrust_test.go b/xslt/streamtrust_test.go new file mode 100644 index 00000000..9b09a249 --- /dev/null +++ b/xslt/streamtrust_test.go @@ -0,0 +1,194 @@ +package xslt + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/knroy/go-xml/v2/xdm" +) + +// The m10 soundness lane's reproducers: stylesheets the §19.8 analysis +// accepted with a known, grounded verdict although a streamed evaluation +// would need data it had already passed. Each must now either be refused +// (a syntactic hole, closed by a spec rule), compile with its verdict marked +// unverified (a declaration hole: the verdict rests on a declaration whose own +// check gave up, or that another module can override), or compile with an +// unknown verdict -- and none may reach the burst path. The "verified" rows +// are controls: the same declarations written so that their checks pass, +// which must not be marked. +func TestStreamTrustReproducers(t *testing.T) { + const hdr = `` + const end = `` + sd := func(body string) string { + return `` + body + `` + } + sdAcc := func(body string) string { + return `` + body + `` + } + fe := func(b string) string { return `` + b + `` } + const mode = `` + apply := sd(``) + const sib = `following-sibling::rec[1]/@k` + + for _, tc := range []struct { + name string + want string // unverified, verified, unknown, or XTSE3430 + files map[string]string // other modules, by name + main string + in string // the module holding the xsl:source-document, if not main + }{ + // Declared-streamable functions whose bodies the check skipped. + {"F1 absorbing fn, where-populated body", "unverified", nil, + hdr + `` + + sd(fe(``)) + end, ""}, + {"F1b absorbing fn, two-instruction body", "unverified", nil, + hdr + `` + + sd(fe(``)) + end, ""}, + {"F1c inspection fn, choose body", "unverified", nil, + hdr + `` + + sd(fe(``)) + end, ""}, + {"F1 control: absorbing fn, modelled body", "verified", nil, + hdr + `` + + sd(fe(``)) + end, ""}, + // A streamable attribute set and accumulators whose checks gave up. + {"F1d streamable attribute set via concat/2", "unverified", nil, + hdr + `` + + sd(fe(``)) + end, ""}, + {"F1d control: motionless attribute set", "verified", nil, + hdr + `` + + sd(fe(``)) + end, ""}, + {"F1e accumulator rule select via concat/2", "unverified", nil, + hdr + `` + + sdAcc(fe(``)) + end, ""}, + {"A1 accumulator rule with a sequence constructor", "unverified", nil, + hdr + `` + + sdAcc(fe(``)) + end, ""}, + {"A1 control: motionless accumulator", "verified", nil, + hdr + `` + + sdAcc(fe(``)) + end, ""}, + // Template rules of a streamable mode that its check did not hold + // to §6.6.4. + {"M1 mode=#all rule", "unverified", nil, + hdr + mode + `` + apply + end, ""}, + {"M2 rule parameter default", "unverified", nil, + hdr + mode + `` + apply + end, ""}, + {"M3 pattern withheld: numeric literal in a comparison", "unverified", nil, + hdr + mode + `` + apply + end, ""}, + {"M3b pattern withheld: rec[x = 1]", "unverified", nil, + hdr + mode + `` + apply + end, ""}, + {"M3d group-starting-with withheld", "unverified", nil, + hdr + sd(``) + end, ""}, + {"M4 rule pattern unknown", "unverified", nil, + hdr + mode + `` + apply + end, ""}, + {"M5 rule body unknown", "unverified", nil, + hdr + mode + `` + apply + end, ""}, + {"M5 control: grounded rule", "verified", nil, + hdr + mode + `` + apply + end, ""}, + // Another module can add rules or override declarations. + {"M6 rule in an included module", "unverified", + map[string]string{"b.xsl": hdr + `` + end}, + hdr + `` + mode + apply + end, ""}, + {"M7 apply-imports into an imported rule", "unverified", + map[string]string{"b.xsl": hdr + `` + end}, + hdr + `` + mode + `` + apply + end, ""}, + {"P1 named template overridden by the importer", "unverified", + map[string]string{"b.xsl": hdr + `` + + sd(fe(``)) + end}, + hdr + `` + end, "b.xsl"}, + {"P2 function overridden by the importer", "unverified", + map[string]string{"b.xsl": hdr + `` + + sd(fe(``)) + end}, + hdr + `` + end, "b.xsl"}, + // The burst path's own shape, which must stay trusted. + {"control: record loop", "verified", nil, + hdr + sd(fe(``)) + end, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + write := func(name, s string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(s), 0o600); err != nil { + t.Fatal(err) + } + } + write("d.xml", `abc`) + for n, s := range tc.files { + write(n, s) + } + _, err := streamTrustRun(t, tc.main, dir) + if tc.want == "XTSE3430" { + if err == nil || !strings.Contains(err.Error(), "XTSE3430") { + t.Fatalf("want XTSE3430, got %v", err) + } + return + } + if err != nil { + t.Fatalf("compile or run: %v", err) + } + if burstRuns != (tc.name == "control: record loop") { + t.Fatalf("burst path taken: %v", burstRuns) + } + // The verdict, read as the burst gate reads it. + src := tc.main + if tc.in != "" { + src = tc.files[tc.in] + } + tree, err := xdm.ParseString(src, xdm.ParseOptions{}) + if err != nil { + t.Fatal(err) + } + var el *xdm.Node + walkElements(firstElement(tree.Root), func(n *xdm.Node) bool { + if isXSL(n, "source-document") { + el = n + } + return el == nil + }) + shared := len(tc.files) > 0 + tr := newDeclTrust(documentElementOf(el), shared) + _, known, unverified := sequenceConstructorVerdict(el, postureStriding, tr.sets, tr.funcs, tr) + got := "verified" + switch { + case !known: + got = "unknown" + case unverified: + got = "unverified" + } + if got != tc.want { + t.Fatalf("verdict %s, want %s", got, tc.want) + } + if burstVerdict(el, shared) != (got == "verified") { + t.Fatalf("burst gate admits %s verdict", got) + } + }) + } +} + +// streamTrustRun compiles src as dir/s.xsl, so that its includes and imports +// resolve, and runs template "main" with bursts enabled. +func streamTrustRun(t *testing.T, src, dir string) (string, error) { + base := fileURIOf(filepath.Join(dir, "s.xsl")) + tree, err := xdm.ParseString(src, xdm.ParseOptions{BaseURI: base}) + if err != nil { + t.Fatal(err) + } + docs, err := NewFileResolver(dir) + if err != nil { + t.Fatal(err) + } + burstRuns = false + sheet, err := Compile(tree.Root, CompileOptions{BaseURI: base, Resolver: docs}) + if err != nil { + return "", err + } + traceBursts = func(string) { burstRuns = true } + defer func() { traceBursts = nil }() + res, err := sheet.Transform(context.Background(), nil, TransformOptions{ + InitialTemplate: "main", Documents: docs}) + if err != nil { + return "", err + } + return res.String(), nil +} From b5703be7a22c79c4d9dad5d2482ffba61306a049 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:02:11 -0500 Subject: [PATCH 30/45] =?UTF-8?q?xslt:=20assess=20an=20xsl:map-entry=20nes?= =?UTF-8?q?ted=20in=20a=20sequence=20constructor=20(=C2=A719.8.4.24)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xsl:map-entry was skipped as if it were an operand of its parent, so one inside an xsl:for-each or xsl:if within xsl:map was never assessed and a streamed node could be bound into the map. It is an instruction, and is now analysed as one. TestXSLT30Suite 11495/23, unchanged. --- xslt/streaminstructions.go | 10 +++++++--- xslt/streamtrust_test.go | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/xslt/streaminstructions.go b/xslt/streaminstructions.go index b9fb8d90..9b43aa9b 100644 --- a/xslt/streaminstructions.go +++ b/xslt/streaminstructions.go @@ -528,6 +528,11 @@ func (a *instrAnalyzer) body(el *xdm.Node) props { // xsl:on-completion, xsl:catch, xsl:matching-substring and the rest are // operands of their parent instruction under its own rule, not members of its // body, and assessing them twice would double-count a consuming operand. +// +// xsl:map-entry is not among them: it is an instruction (§19.8.4.24), and +// one nested in an xsl:for-each or xsl:if inside an xsl:map is a member of +// that instruction's body, where skipping it let a streamed node be bound into +// the map unseen. func isSequenceConstructorExcluded(el *xdm.Node) bool { if el.Name().URI != xdm.NSXSL { return false @@ -535,7 +540,7 @@ func isSequenceConstructorExcluded(el *xdm.Node) bool { switch el.Name().Local { case "param", "sort", "with-param", "on-completion", "catch", "matching-substring", "non-matching-substring", "merge-source", - "merge-key", "merge-action", "when", "otherwise", "map-entry", + "merge-key", "merge-action", "when", "otherwise", "context-item", "fallback": return true } @@ -1503,8 +1508,7 @@ func (a *instrAnalyzer) mapInstruction(el *xdm.Node) props { return props{postureGrounded, widest} } // Otherwise the posture and sweep are those of the contained sequence - // constructor -- which, because xsl:map-entry is excluded from a - // sequence constructor, must be assessed with the entries included. + // constructor, entries included. var ops []operand for _, c := range el.ChildElements() { ops = append(ops, operand{ diff --git a/xslt/streamtrust_test.go b/xslt/streamtrust_test.go index 9b09a249..bb5da867 100644 --- a/xslt/streamtrust_test.go +++ b/xslt/streamtrust_test.go @@ -102,6 +102,11 @@ func TestStreamTrustReproducers(t *testing.T) { map[string]string{"b.xsl": hdr + `` + sd(fe(``)) + end}, hdr + `` + end, "b.xsl"}, + // Operands the analysis skipped, now assessed by their own rules. + {"S1 xsl:map-entry nested in xsl:for-each inside xsl:map", "XTSE3430", nil, + hdr + sd(``) + end, ""}, + {"S1 control: xsl:map-entry with a grounded value", "verified", nil, + hdr + sd(``) + end, ""}, // The burst path's own shape, which must stay trusted. {"control: record loop", "verified", nil, hdr + sd(fe(``)) + end, ""}, From 8c391d84bed8367f2529201318784b9ca8f6ced9 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:03:42 -0500 Subject: [PATCH 31/45] =?UTF-8?q?xslt:=20assess=20an=20extension=20instruc?= =?UTF-8?q?tion=20by=20its=20xsl:fallback=20children=20(=C2=A719.8.4.2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An element in an extension namespace was analysed as a literal result element, whose excluded xsl:fallback children were never looked at -- yet fallback is what this processor runs. Its operands are now the fallback sequence constructors, usage transmission. TestXSLT30Suite 11495/23, unchanged. --- xslt/streaminstructions.go | 13 +++++++++++++ xslt/streamtrust_test.go | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/xslt/streaminstructions.go b/xslt/streaminstructions.go index 9b43aa9b..a4fde370 100644 --- a/xslt/streaminstructions.go +++ b/xslt/streaminstructions.go @@ -995,6 +995,19 @@ func (a *instrAnalyzer) absorbingSelectOrBody(el *xdm.Node, avts []string) props // literalResultElement applies §19.8.4.1. func (a *instrAnalyzer) literalResultElement(el *xdm.Node) props { var ops []operand + // §19.8.4.2: an element in an extension namespace is an instruction, not + // a literal result element, and this processor recognises none + // (compileLiteralElement), so its operands are the sequence constructors + // of its xsl:fallback children, with usage transmission -- which is what + // runs. + if isExtensionInstruction(el) { + for _, c := range el.ChildElements() { + if isXSL(c, "fallback") { + ops = append(ops, a.bodyOperand(c, usageTransmission)) + } + } + return combine(ops, false) + } for at := range el.Attrs() { // xsl:use-attribute-sets is handled below, and is not itself an // attribute value template. diff --git a/xslt/streamtrust_test.go b/xslt/streamtrust_test.go index bb5da867..2600191d 100644 --- a/xslt/streamtrust_test.go +++ b/xslt/streamtrust_test.go @@ -107,6 +107,10 @@ func TestStreamTrustReproducers(t *testing.T) { hdr + sd(``) + end, ""}, {"S1 control: xsl:map-entry with a grounded value", "verified", nil, hdr + sd(``) + end, ""}, + {"S2 an extension instruction's xsl:fallback", "XTSE3430", nil, + hdr + sd(fe(``)) + end, ""}, + {"S2 control: a grounded xsl:fallback", "verified", nil, + hdr + sd(fe(``)) + end, ""}, // The burst path's own shape, which must stay trusted. {"control: record loop", "verified", nil, hdr + sd(fe(``)) + end, ""}, From ff0ed5e5c3d6b85698ef1c710b9233cb16e766dc Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:05:01 -0500 Subject: [PATCH 32/45] =?UTF-8?q?xslt:=20count=20xsl:result-document's=20s?= =?UTF-8?q?erialization=20AVTs=20as=20operands=20(=C2=A719.8.4.35)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rule's comment said every serialization attribute was treated as an absorbing AVT operand; the code read href alone, so a consuming expression in item-separator, say, was a second read of the stream that went unseen. TestXSLT30Suite 11495/23, unchanged. --- xslt/streaminstructions.go | 12 ++++++++++-- xslt/streamtrust_test.go | 6 +++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/xslt/streaminstructions.go b/xslt/streaminstructions.go index a4fde370..e2f84caa 100644 --- a/xslt/streaminstructions.go +++ b/xslt/streaminstructions.go @@ -632,9 +632,17 @@ func (a *instrAnalyzer) instruction(el *xdm.Node) props { case "result-document": // §19.8.4.35: href and the serialization AVTs absorb, as does the // body. The serialization attributes are all AVTs; treating every - // attribute other than the ones with expression values as an AVT - // covers them without listing forty names. + // attribute other than the few that are not as an AVT covers them + // without listing forty names. ops := a.avtOperands(el, "href") + for at := range el.Attrs() { + switch n := at.Name(); { + case n.URI != "", standardAttributes[n.Local], n.Local == "href", + n.Local == "validation", n.Local == "type", n.Local == "use-character-maps": + default: + ops = append(ops, a.avtOperands(el, n.Local)...) + } + } ops = append(ops, a.bodyOperand(el, usageAbsorption)) return combine(ops, false) diff --git a/xslt/streamtrust_test.go b/xslt/streamtrust_test.go index 2600191d..6263b910 100644 --- a/xslt/streamtrust_test.go +++ b/xslt/streamtrust_test.go @@ -108,9 +108,13 @@ func TestStreamTrustReproducers(t *testing.T) { {"S1 control: xsl:map-entry with a grounded value", "verified", nil, hdr + sd(``) + end, ""}, {"S2 an extension instruction's xsl:fallback", "XTSE3430", nil, - hdr + sd(fe(``)) + end, ""}, + hdr + sd(fe(``)) + end, ""}, {"S2 control: a grounded xsl:fallback", "verified", nil, hdr + sd(fe(``)) + end, ""}, + {"S3 xsl:result-document serialization AVT", "XTSE3430", nil, + hdr + sd(``) + end, ""}, + {"S3 control: literal serialization attributes", "verified", nil, + hdr + sd(``) + end, ""}, // The burst path's own shape, which must stay trusted. {"control: record loop", "verified", nil, hdr + sd(fe(``)) + end, ""}, From e94cdf9c9faa927b231c2d9bfe0cf9d2727c1e2f Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:06:34 -0500 Subject: [PATCH 33/45] xslt: withhold the verdict on an inline function that closes over a streamed node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §19.8.8.16 grounds every inline function that names no streaming parameter. Two closures escape that test: one over a range variable this analysis's quantified-expression data flow bound to streamed nodes, and one calling fn:current(), whose value is the streamed item current at creation. Both are now unknown -- withheld, not refused. TestXSLT30Suite 11495/23, unchanged. --- xslt/streamexprs.go | 37 +++++++++++++++++++++++++++++++++++-- xslt/streamtrust_test.go | 11 +++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/xslt/streamexprs.go b/xslt/streamexprs.go index 7dc5890b..01c2f968 100644 --- a/xslt/streamexprs.go +++ b/xslt/streamexprs.go @@ -429,10 +429,34 @@ func (a *analyzer) withVar(name xdm.QName, p props) *analyzer { // access a streamed node is by having the streamed node in its closure, and // this is prevented by the rule above". A walker that stopped at a nested // function boundary would let exactly that closure through. +// +// Two closures the section does not foresee are unknown rather than grounded. +// A range variable that this analysis's own data flow (quantifiedExpr) bound +// to streamed nodes is not a streaming parameter, but closing over it carries +// the streamed node out exactly as the note fears. And fn:current() in the +// body denotes the item that was current where the function was created -- +// a streamed node in a streamable construct -- which no variable reference +// betrays. The spec read literally grounds both; the verdict is withheld +// rather than refused, because the first rests on data flow the spec does +// not have and the second on a gap in its rules. func (a *analyzer) inlineFunction(x *xpath.InlineFunctionExpr) props { + for name, p := range a.vars { + if p.posture == postureGrounded { + continue + } + b := *a + b.hasStreamParam, b.streamingParam = true, name + if b.bodyMentionsStreamingParam(x.Body) != mentionNo { + return a.unknown() + } + } if !a.hasStreamParam { // With no streaming parameter in scope there is nothing the body - // could refer to, so the second sentence applies directly. + // could refer to, so the second sentence applies directly -- once + // the walk has found no call on fn:current(). + if a.bodyMentionsStreamingParam(x.Body) == mentionUnknown { + return a.unknown() + } return groundedMotionless } switch a.bodyMentionsStreamingParam(x.Body) { @@ -487,8 +511,12 @@ func (a *analyzer) bodyMentionsStreamingParam(e xpath.Expr) mention { case *xpath.Literal, *xpath.ContextItem: return mentionNo - // A name, not a reference to a variable. + // A name, not a reference to a variable -- except fn:current#0, which + // reaches the creating context's current item as a call does. case *xpath.NamedFunctionRef: + if x.Name.URI == fnNS && x.Name.Local == "current" { + return mentionUnknown + } return mentionNo // Nested, and still within this declaration's text -- which is the point. @@ -505,6 +533,11 @@ func (a *analyzer) bodyMentionsStreamingParam(e xpath.Expr) mention { return a.mentionAny(append(bindingExprs(x.Bindings), x.Test)) case *xpath.FuncCall: + // See inlineFunction: no variable reference, and still a streamed + // node. + if x.Name.URI == fnNS && x.Name.Local == "current" { + return mentionUnknown + } return a.mentionAny(x.Args) case *xpath.DynamicCall: return a.mentionAny(append([]xpath.Expr{x.Target}, x.Args...)) diff --git a/xslt/streamtrust_test.go b/xslt/streamtrust_test.go index 6263b910..18f7a03f 100644 --- a/xslt/streamtrust_test.go +++ b/xslt/streamtrust_test.go @@ -115,6 +115,17 @@ func TestStreamTrustReproducers(t *testing.T) { hdr + sd(``) + end, ""}, {"S3 control: literal serialization attributes", "verified", nil, hdr + sd(``) + end, ""}, + // Closures over a streamed node the inline-function rule does not see. + {"S4 inline function closes over a quantified range variable", "unknown", nil, + hdr + sd(``) + end, ""}, + {"S4b closure returns ../rec[last()]", "unknown", nil, + hdr + sd(``) + end, ""}, + {"S4 control: closure over a grounded range variable", "verified", nil, + hdr + sd(``) + end, ""}, + {"X1 inline function body calls current()", "unknown", nil, + hdr + sd(fe(``)) + end, ""}, + {"X1 control: inline function with no closure", "verified", nil, + hdr + sd(fe(``)) + end, ""}, // The burst path's own shape, which must stay trusted. {"control: record loop", "verified", nil, hdr + sd(fe(``)) + end, ""}, From ca1b02925f9e55343b813d6025a57728666f246b Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:08:33 -0500 Subject: [PATCH 34/45] =?UTF-8?q?xslt:=20judge=20a=20boolean=20pattern=20p?= =?UTF-8?q?redicate=20by=20the=20motionless=20test=20(=C2=A719.8.10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §19.8.10 calls a predicate positional when its static type is numeric or it calls position(), last() or function-lookup(); the classifier's numeric test is syntactic. Its false positives on comparisons were withheld from refusal, which let rec[following-sibling::rec/@k = 3] through unjudged in a rule pattern or a group-starting-with. A boolean-typed predicate calling none of the three is now non-positional and takes the motionless test like any other; the withholding goes. TestXSLT30Suite 11495/23, unchanged. --- xslt/streamaccumulators.go | 18 ++++++-- xslt/streamcontext.go | 86 ++------------------------------------ xslt/streaminstructions.go | 8 +--- xslt/streamtrust_test.go | 8 ++-- 4 files changed, 25 insertions(+), 95 deletions(-) diff --git a/xslt/streamaccumulators.go b/xslt/streamaccumulators.go index e6af448b..e996e67f 100644 --- a/xslt/streamaccumulators.go +++ b/xslt/streamaccumulators.go @@ -429,10 +429,20 @@ func patternStepFreeRanging(e xpath.Expr, at *xdm.Node, matchedAllowsChildren bo func patternPredicatesFreeRanging(preds []xpath.Expr, allowsChildren, matchedAllowsChildren bool) (bool, bool) { for _, pred := range preds { // Condition (b), second half: the predicate must be non-positional. - // isPositionalPredicate already implements §19.8.10's definition -- - // a call to position(), last(), or function-lookup() outside a - // nested predicate, or a numeric predicate expression. - if isPositionalPredicate(pred) { + // §19.8.10 defines that by a call to position(), last() or + // function-lookup() outside a nested predicate, or by the + // predicate's STATIC TYPE being numeric. isPositionalPredicate + // answers a syntactic question instead -- "does a numeric literal + // appear anywhere?" -- which over-reaches on a comparison: + // §19.8.10's own list calls "p[@status = $status-codes[1]]" + // motionless, its type being xs:boolean, and accumulator-055s + // matches "p/text()[accumulator-before('text-in-p-count') eq 1]". + // So a boolean-typed predicate that calls none of the three is + // non-positional, and goes on to the motionless test below like any + // other. (It used to be withheld instead, which let + // "rec[following-sibling::rec/@k = 3]" through unjudged.) + if isPositionalPredicate(pred) && + (callsPositionOrLast(pred) || !hasBooleanStaticType(pred)) { return true, true } // Condition (b), first half: the expression immediately contained in diff --git a/xslt/streamcontext.go b/xslt/streamcontext.go index 648cd31f..aaccec75 100644 --- a/xslt/streamcontext.go +++ b/xslt/streamcontext.go @@ -78,7 +78,7 @@ func checkStreamableModePatterns(root *xdm.Node) error { return true } free, known := patternIsFreeRanging(a.Value(), el) - if known && free && !patternPredicateOnlySyntacticallyNumeric(a.Value(), el) { + if known && free { err = fmt.Errorf( "the match pattern %q of a template rule in a streamable "+ "mode is not motionless, so it is not "+ @@ -114,83 +114,6 @@ func streamableModeNames(root *xdm.Node) map[string]bool { return out } -// patternPredicateOnlySyntacticallyNumeric reports whether a pattern's -// free-ranging verdict rests only on isPositionalPredicate's deliberately -// over-broad numeric test, and so must not be raised as an error. -// -// §19.8.10 defines a non-positional predicate by the predicate expression's -// STATIC TYPE: it is non-numeric when the intersection of that type with -// U{xs:decimal, xs:double, xs:float} is empty. isPositionalPredicate answers a -// different, syntactic question -- "does a numeric literal appear anywhere -// inside?" -- and §19.8.8.8, its original caller, can afford that: there an -// over-broad answer only withholds the scanning-expression rescue, which loses -// precision and reports nothing. -// -// Here the same answer would be reported as XTSE3430, so the gap between the -// two questions becomes a spurious rejection. accumulator-055s writes -// -// match="p/text()[accumulator-before('text-in-p-count') eq 1]" -// -// whose predicate is a comparison. §19.8.10's own list makes the distinction -// explicitly: "p[@status = $status-codes[1]]" is motionless even though a -// numeric literal appears in it, because the predicate's static type is -// xs:boolean, while "p[$pnum + 1]" is not, because its type is numeric. -// -// So a predicate whose top-level operator is a comparison or a boolean -// connective has a boolean static type, and no numeric literal buried in its -// operands can change that. Where the free-ranging verdict rests on nothing -// but such a predicate, the classification is withheld rather than reported -- -// the same "no opinion" the analysis gives any construct it does not model. -func patternPredicateOnlySyntacticallyNumeric(src string, at *xdm.Node) bool { - ns := newNSResolver(at, xpathDefaultNamespace(at)) - expr, err := xpath.ParseVersion(src, ns, xpathVersionAt(at)) - if err != nil { - return false - } - var preds []xpath.Expr - switch x := expr.(type) { - case *xpath.Step: - preds = x.Predicates - case *xpath.PathExpr: - if len(x.Steps) == 0 { - return false - } - step, ok := x.Steps[len(x.Steps)-1].(*xpath.Step) - if !ok { - return false - } - preds = step.Predicates - case *xpath.FilterExpr: - // The XSLT 3.0 ".[...]" pattern, which patternExprFreeRanging now - // classifies. Its predicates are top-level pattern predicates, so - // they need the same rescue as a step's. - if _, ok := x.Base.(*xpath.ContextItem); !ok { - return false - } - preds = x.Predicates - default: - return false - } - sawRescued := false - for _, pred := range preds { - if !isPositionalPredicate(pred) { - continue - } - // §19.8.10's first condition is independent of the static type: a - // call to position(), last() or function-lookup() outside a nested - // predicate makes the predicate positional whatever it returns. - // "p[position() gt 2]" is a comparison, and the spec still lists it - // as not motionless. - if callsPositionOrLast(pred) || !hasBooleanStaticType(pred) { - // A genuinely positional predicate -- "p[1]", "p[last()]" -- so - // the verdict stands on its own merits. - return false - } - sawRescued = true - } - return sawRescued -} - // callsPositionOrLast reports whether an expression calls one of the three // functions §19.8.10 names as making a predicate positional: fn:position, // fn:last and fn:function-lookup. @@ -360,9 +283,8 @@ func checkStreamableModeBodies(root *xdm.Node) error { // there would silence a refusal §19.8.4.19 derived on its own. // // Both tests are deliberately the ones forEachGroup applies, so the two cannot -// disagree: the same patternIsFreeRanging with the same -// patternPredicateOnlySyntacticallyNumeric rescue, the same key assessed in -// the select's own context posture, and the same restriction to a non-grounded +// disagree: the same patternIsFreeRanging, the same key assessed in the +// select's own context posture, and the same restriction to a non-grounded // select -- which is what keeps si-group-203 running, its select being the // grounded "//Item/copy-of()". func hasIndependentGroupRefusal(rule *xdm.Node) bool { @@ -385,7 +307,7 @@ func hasIndependentGroupRefusal(rule *xdm.Node) bool { } if at := groupPatternAttr(el); at != nil { free, known := patternIsFreeRanging(at.Value(), el) - if known && free && !patternPredicateOnlySyntacticallyNumeric(at.Value(), el) { + if known && free { found = true return false } diff --git a/xslt/streaminstructions.go b/xslt/streaminstructions.go index e2f84caa..656e1804 100644 --- a/xslt/streaminstructions.go +++ b/xslt/streaminstructions.go @@ -2065,16 +2065,12 @@ func (a *instrAnalyzer) forEachGroup(el *xdm.Node) props { // §19.8.10 calls free-ranging on the "p[b]" rule; the catalog asserts // OUTPUT for it, and it is right to, because the snapshot is in memory. // - // The verdict is taken only when patternIsFreeRanging is sure of it, and - // only when the free-ranging answer does not rest on the deliberately - // over-broad numeric test that patternPredicateOnlySyntacticallyNumeric - // identifies -- the same withholding checkStreamableModePatterns applies - // to a rule's match pattern, and for the same reason. + // The verdict is taken only when patternIsFreeRanging is sure of it. if p := groupPatternAttr(el); p != nil && sel.posture != postureGrounded { free, known := patternIsFreeRanging(p.Value(), el) if !known { a.known = false - } else if free && !patternPredicateOnlySyntacticallyNumeric(p.Value(), el) { + } else if free { return roamingFreeRanging } if !patternVerified(p.Value(), el) { diff --git a/xslt/streamtrust_test.go b/xslt/streamtrust_test.go index 18f7a03f..92e5f490 100644 --- a/xslt/streamtrust_test.go +++ b/xslt/streamtrust_test.go @@ -75,12 +75,14 @@ func TestStreamTrustReproducers(t *testing.T) { hdr + mode + `` + apply + end, ""}, {"M2 rule parameter default", "unverified", nil, hdr + mode + `` + apply + end, ""}, - {"M3 pattern withheld: numeric literal in a comparison", "unverified", nil, + {"M3 pattern withheld: numeric literal in a comparison", "XTSE3430", nil, hdr + mode + `` + apply + end, ""}, - {"M3b pattern withheld: rec[x = 1]", "unverified", nil, + {"M3b pattern withheld: rec[x = 1]", "XTSE3430", nil, hdr + mode + `` + apply + end, ""}, - {"M3d group-starting-with withheld", "unverified", nil, + {"M3d group-starting-with withheld", "XTSE3430", nil, hdr + sd(``) + end, ""}, + {"M3 control: comparison with a numeric literal, motionless", "verified", nil, + hdr + mode + `` + apply + end, ""}, {"M4 rule pattern unknown", "unverified", nil, hdr + mode + `` + apply + end, ""}, {"M5 rule body unknown", "unverified", nil, From 268acceadc534e45c57ed294e25aec952e19a733 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:10:01 -0500 Subject: [PATCH 35/45] =?UTF-8?q?xslt:=20hold=20mode=3D"#all"=20template?= =?UTF-8?q?=20rules=20to=20a=20streamable=20mode=20(=C2=A76.6.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §6.6.4 applies the streamable-template conditions to every rule that includes the mode, "including by specifying #all". templateModeNames names modes and leaves #all out, so such a rule was never checked, and a comment said that was deliberate. Both the pattern and the body checks now include it. TestXSLT30Suite 11495/23, unchanged. --- xslt/streamcontext.go | 44 ++++++++++++++++++++-------------------- xslt/streamtrust_test.go | 4 +++- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/xslt/streamcontext.go b/xslt/streamcontext.go index aaccec75..76dc1895 100644 --- a/xslt/streamcontext.go +++ b/xslt/streamcontext.go @@ -60,21 +60,12 @@ func checkStreamableModePatterns(root *xdm.Node) error { if !isXSL(el, "template") { return true } - // templateModeNames returns nil for a named template (no @match) and - // for "#all"/"#current", neither of which names a mode that can be - // looked up. A rule with no match pattern has no pattern to judge. + // A rule with no match pattern has no pattern to judge. a := el.Attr("", "match") if a == nil { return true } - inStreamable := false - for _, m := range templateModeNames(el) { - if streamable[m] { - inStreamable = true - break - } - } - if !inStreamable { + if !inStreamableMode(el, streamable) { return true } free, known := patternIsFreeRanging(a.Value(), el) @@ -90,6 +81,21 @@ func checkStreamableModePatterns(root *xdm.Node) error { return err } +// inStreamableMode reports whether a template rule belongs to a mode the +// module declares streamable. mode="#all" belongs to every mode (§6.6.4), +// which templateModeNames, naming modes, leaves out. +func inStreamableMode(el *xdm.Node, streamable map[string]bool) bool { + if len(streamable) > 0 && strings.Contains(" "+el.AttrValue("mode")+" ", " #all ") { + return true + } + for _, m := range templateModeNames(el) { + if streamable[m] { + return true + } + } + return false +} + // streamableModeNames collects the Clark names of the modes an xsl:mode // declaration marks streamable="yes". // @@ -198,9 +204,10 @@ func hasBooleanStaticType(e xpath.Expr) bool { // decidable: "apply-templates select='.//section'" is free-ranging because // .//section descends, whatever the section rule then goes on to do. // -// A rule whose @mode is "#all" is not assessed. templateModeNames excludes it, -// because "#all" names every mode rather than any particular one, and a rule -// written for every mode cannot be held to the streamability of one of them. +// A rule whose @mode is "#all" is assessed too: §6.6.4 holds to the rules +// every template rule "that includes that mode (explicitly or implicitly, +// including by specifying #all)", and §19.10 makes the consequences the same. +// (This comment once said the opposite, and such a rule was never checked.) // // The verdict is withheld unless the analysis fully modelled the body (known), // which is the guard the whole check rests on, and unless it rests on @@ -225,14 +232,7 @@ func checkStreamableModeBodies(root *xdm.Node) error { if !isXSL(el, "template") || el.Attr("", "match") == nil { return true } - inStreamable := false - for _, m := range templateModeNames(el) { - if streamable[m] { - inStreamable = true - break - } - } - if !inStreamable { + if !inStreamableMode(el, streamable) { return true } p, known := analyzeSequenceConstructor(el, postureStriding, sets, funcs) diff --git a/xslt/streamtrust_test.go b/xslt/streamtrust_test.go index 92e5f490..2566d8df 100644 --- a/xslt/streamtrust_test.go +++ b/xslt/streamtrust_test.go @@ -71,8 +71,10 @@ func TestStreamTrustReproducers(t *testing.T) { sdAcc(fe(``)) + end, ""}, // Template rules of a streamable mode that its check did not hold // to §6.6.4. - {"M1 mode=#all rule", "unverified", nil, + {"M1 mode=#all rule", "XTSE3430", nil, hdr + mode + `` + apply + end, ""}, + {"M1 control: a grounded mode=#all rule", "verified", nil, + hdr + mode + `` + apply + end, ""}, {"M2 rule parameter default", "unverified", nil, hdr + mode + `` + apply + end, ""}, {"M3 pattern withheld: numeric literal in a comparison", "XTSE3430", nil, From 79206352b362e9fa83a5130cb0d5fe572ef56ae7 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:11:28 -0500 Subject: [PATCH 36/45] =?UTF-8?q?xslt:=20require=20a=20streamable=20rule's?= =?UTF-8?q?=20parameter=20defaults=20to=20be=20motionless=20(=C2=A76.6.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §6.6.4's third condition on a template rule of a streamable mode is that every expression in a contained xsl:param is motionless. It was not checked, so a default reading a following sibling compiled. TestXSLT30Suite 11495/23, unchanged. --- xslt/streamcontext.go | 33 +++++++++++++++++++++++++++++++++ xslt/streamtrust.go | 10 +--------- xslt/streamtrust_test.go | 4 +++- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/xslt/streamcontext.go b/xslt/streamcontext.go index 76dc1895..00b20601 100644 --- a/xslt/streamcontext.go +++ b/xslt/streamcontext.go @@ -235,6 +235,23 @@ func checkStreamableModeBodies(root *xdm.Node) error { if !inStreamableMode(el, streamable) { return true } + // §6.6.4's third condition: "Every expression and contained + // sequence constructor in a contained xsl:param element (the + // construct that provides the default value of the parameter) is + // motionless." + for _, c := range el.ChildElements() { + if !isXSL(c, "param") { + continue + } + if p, known, _ := ruleParamDefault(c, sets, funcs, nil); known && p.sweep != sweepMotionless { + err = fmt.Errorf( + "the default of parameter %q of the template rule matching %q "+ + "in a streamable mode is %v, but §6.6.4 requires it to be "+ + "motionless, so it is not guaranteed-streamable (XTSE3430)", + c.AttrValue("name"), el.AttrValue("match"), p.sweep) + return false + } + } p, known := analyzeSequenceConstructor(el, postureStriding, sets, funcs) if !known || (bodyCallsCurrentGroup(el) && !hasIndependentGroupRefusal(el)) { return true @@ -266,6 +283,22 @@ func checkStreamableModeBodies(root *xdm.Node) error { return err } +// ruleParamDefault assesses the default of a template rule's xsl:param, +// which is evaluated with the rule's own focus: striding, by §19.6. +func ruleParamDefault( + param *xdm.Node, sets map[xdm.QName][]*xdm.Node, funcs map[funcKey]*streamFunc, trust *declTrust, +) (props, bool, bool) { + a := &instrAnalyzer{ctxPosture: postureStriding, ctxAllowsChildren: true, + known: true, attrSets: sets, funcs: funcs, trust: trust} + var p props + if s := param.Attr("", "select"); s != nil { + p = a.exprProps(s.Value(), param) + } else { + p = a.body(param) + } + return p, a.known, a.unverified +} + // hasIndependentGroupRefusal reports whether some xsl:for-each-group in the // rule is refused by §19.8.4.19 for a reason that has nothing to do with the // call on fn:current-group -- either a grouping pattern §19.8.10 classifies as diff --git a/xslt/streamtrust.go b/xslt/streamtrust.go index 4ffef4ed..565bc23e 100644 --- a/xslt/streamtrust.go +++ b/xslt/streamtrust.go @@ -278,15 +278,7 @@ func (t *declTrust) ruleOK(rule *xdm.Node) bool { if !isXSL(c, "param") { continue } - a := &instrAnalyzer{ctxPosture: postureStriding, ctxAllowsChildren: true, - known: true, attrSets: t.sets, funcs: t.funcs, trust: t} - var p props - if s := c.Attr("", "select"); s != nil { - p = a.exprProps(s.Value(), c) - } else { - p = a.body(c) - } - if !a.known || a.unverified || p.sweep != sweepMotionless { + if p, known, unverified := ruleParamDefault(c, t.sets, t.funcs, t); !known || unverified || p.sweep != sweepMotionless { return false } } diff --git a/xslt/streamtrust_test.go b/xslt/streamtrust_test.go index 2566d8df..c553be65 100644 --- a/xslt/streamtrust_test.go +++ b/xslt/streamtrust_test.go @@ -75,8 +75,10 @@ func TestStreamTrustReproducers(t *testing.T) { hdr + mode + `` + apply + end, ""}, {"M1 control: a grounded mode=#all rule", "verified", nil, hdr + mode + `` + apply + end, ""}, - {"M2 rule parameter default", "unverified", nil, + {"M2 rule parameter default", "XTSE3430", nil, hdr + mode + `` + apply + end, ""}, + {"M2 control: a motionless parameter default", "verified", nil, + hdr + mode + `` + apply + end, ""}, {"M3 pattern withheld: numeric literal in a comparison", "XTSE3430", nil, hdr + mode + `` + apply + end, ""}, {"M3b pattern withheld: rec[x = 1]", "XTSE3430", nil, From 69bffdb019e8507cdfc58e3c3d5f636e4abe5bc5 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:11:50 -0500 Subject: [PATCH 37/45] xslt: test the soundness reproducers in the burst path's own shape Three of the m10 reproducers rewritten as the one record loop the burst gate plans for, where its whitelist is the only other guard: a nested xsl:map-entry is refused, and the two closures over a range variable are unknown and stay on the tree path. --- xslt/streamtrust_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/xslt/streamtrust_test.go b/xslt/streamtrust_test.go index c553be65..92b2d3fc 100644 --- a/xslt/streamtrust_test.go +++ b/xslt/streamtrust_test.go @@ -132,6 +132,14 @@ func TestStreamTrustReproducers(t *testing.T) { hdr + sd(fe(``)) + end, ""}, {"X1 control: inline function with no closure", "verified", nil, hdr + sd(fe(``)) + end, ""}, + // The same holes in the burst path's own shape: one loop over the + // records, where the gate's whitelist is the only other guard. + {"B1 nested xsl:map-entry inside the record loop", "XTSE3430", nil, + hdr + sd(fe(``)) + end, ""}, + {"B2 closure over a range variable inside the record loop", "unknown", nil, + hdr + sd(fe(``)) + end, ""}, + {"B3 closure reaching only an ancestor", "unknown", nil, + hdr + sd(fe(``)) + end, ""}, // The burst path's own shape, which must stay trusted. {"control: record loop", "verified", nil, hdr + sd(fe(``)) + end, ""}, From ec90a839ac1c454acfd0f5ff89562e54416d551f Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:18:24 -0500 Subject: [PATCH 38/45] =?UTF-8?q?docs:=20stage=200=20of=20streaming=20?= =?UTF-8?q?=E2=80=94=20verdicts=20carry=20an=20unverified=20bit,=20six=20c?= =?UTF-8?q?onstructs=20now=20judged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 15 +++++++++++++++ README.md | 4 ++-- docs/conformance-gaps.md | 2 +- docs/known-gaps.md | 21 ++++++++++++++++++--- docs/stats.md | 2 +- docs/testing.md | 2 +- docs/todo.md | 2 +- 7 files changed, 39 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c176a25..9b3a6c5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,13 @@ is byte-identical; new API is additive. | `xslt.FileResolver.MaxNodes`; `MaxBytes` reaches the parser | Raising `MaxBytes` past 64 MB had no effect on `fn:doc`, the parser kept its own default. Both limits now apply to what the resolver parses. | [`80f999f`][80f999f] | | `xdmbuild.InPlaceDocument` | A builder rooted at a new document node, for content built straight into it. | [`302c932`][302c932] | +### Changed — streamability analysis + +| Change | Problem → solution | Commit | +|---|---|---| +| Six constructs escaped the §19.8 check and compiled | A nested `xsl:map-entry`, extension instructions (their `xsl:fallback`), `xsl:result-document` serialization AVTs, boolean pattern predicates, `mode="#all"` rules and rule `xsl:param` defaults are now judged: such stylesheets get XTSE3430. | [`b5703be`][b5703be], [`8c391d8`][8c391d8], [`ff0ed5e`][ff0ed5e], [`ca1b029`][ca1b029], [`268acce`][268acce], [`7920635`][7920635] | +| A verdict could rest on a declaration nobody checked | Verdicts carry an unverified bit for streamable functions, attribute sets, accumulators, modes and other modules; streamed execution requires it clear. No refusal changes. | [`b12ecbf`][b12ecbf], [`e94cdf9`][e94cdf9] | + ### Changed — memory | Change | Problem → solution | Commit | @@ -1869,3 +1876,11 @@ here so every entry in this file sits under a release. [b4cd614]: https://github.com/knroy/go-xml/commit/b4cd614 [907c674]: https://github.com/knroy/go-xml/commit/907c674 [c7952b0]: https://github.com/knroy/go-xml/commit/c7952b0 +[b5703be]: https://github.com/knroy/go-xml/commit/b5703be +[8c391d8]: https://github.com/knroy/go-xml/commit/8c391d8 +[ff0ed5e]: https://github.com/knroy/go-xml/commit/ff0ed5e +[ca1b029]: https://github.com/knroy/go-xml/commit/ca1b029 +[268acce]: https://github.com/knroy/go-xml/commit/268acce +[7920635]: https://github.com/knroy/go-xml/commit/7920635 +[b12ecbf]: https://github.com/knroy/go-xml/commit/b12ecbf +[e94cdf9]: https://github.com/knroy/go-xml/commit/e94cdf9 diff --git a/README.md b/README.md index 075ce56e..447dca25 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ and maintains it as a project of his own. | **XSD 1.1** | 99.98% instance (26,217 of 26,222); **99.97%** schema-validity (15,350 of 15,354); opt-in via `Version11` | | **RELAX NG** | 100.00% of James Clark's spectest (965 of 965 assertions); XML and compact syntax, from Go and from `go-xml validate -rng` | | **DTD** | content models, attribute defaults, enumerations, `ID`/`IDREF`; external subset, parameter entities across both subsets, conditional sections — via `dtd.Load` with a caller-supplied resolver, nothing fetched by default | -| **Tests** | 2,984 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | +| **Tests** | 2,986 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | | **Production schemas** | UBL 2.1, UN/CEFACT CII, Factur-X/ZUGFeRD, Peppol BIS 3.0 — 88 schemas load, instances validate clean | | **API** | 1.2; the exported surface is stable and additive over 1.1, and a breaking change means 2.0 with a new module path | @@ -1391,7 +1391,7 @@ back, is in [docs/testing.md](docs/testing.md). | method | what it catches | what it misses | |---|---|---| -| **Unit tests** (2,984 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** (2,986 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | | **Spec inventories** | features absent entirely | features present but behaving wrongly | | **Saxon differential** | subtle behavioural divergence on real stylesheets | constructs the corpora do not use | | **W3C QT3 suite** | systematic conformance across 22,054 XPath and 30,517 XQuery cases | XSLT (it is an XPath suite) | diff --git a/docs/conformance-gaps.md b/docs/conformance-gaps.md index 996ccd03..3bc71a60 100644 --- a/docs/conformance-gaps.md +++ b/docs/conformance-gaps.md @@ -49,7 +49,7 @@ W3C disagreements: 0 + 0 + 0 + 1 + 8 + 23 + 30 + 31 + 0 = 93. Measured 2026-09-1 -The unit-test suite is 2,984 tests. +The unit-test suite is 2,986 tests. The last two rows are not W3C suites but real-world corpora — DocBook xslTNG's diff --git a/docs/known-gaps.md b/docs/known-gaps.md index cee26a9d..85a50409 100644 --- a/docs/known-gaps.md +++ b/docs/known-gaps.md @@ -182,7 +182,8 @@ Measured on branches in 2026-11 and not adopted, each with the reason: Since v2.2.0 an `xsl:source-document streamable="yes"` whose body is one `xsl:for-each` or `xsl:iterate` (optionally inside constant literal result elements) over a child path of plain names, with non-positional predicates on -the last step only, and a body the gate recognises, runs record by record +the last step only, and a body the gate recognises with a known and verified +§19.8 verdict, runs record by record through `xdm.ParseBursts`: a 1 GB file totalled by `xsl:iterate` peaks at 27 MB RSS against 3.7 GB, at about 2× the CPU. Everything else builds the tree, as before. In the W3C suite 9 of the 2,599 streamable source documents take the @@ -246,7 +247,13 @@ reports whether it *modelled* every construct it met separately from what it concluded, and an error is raised only on a fully-modelled verdict; anything else is "no opinion". A missing rule leaves a case failing, while a wrong rule would reject a valid stylesheet, and it is the second that the whole-corpus -scan measures at zero. +scan measures at zero. Since v2.2.0 a verdict also carries an *unverified* bit: +it is set when the verdict rests on a streamable function, attribute set, +accumulator or mode whose own check gave up, or on a declaration in another +module. Refusals ignore the bit, so no XTSE3430 result moves; streamed execution +requires it clear. In the W3C streaming stylesheets the static checks accept, +58 known, grounded containers in 41 files are unverified (modes 28, functions +10, cross-module 10, accumulators 7, other 3). **All three are measured, and each known fix costs more than it gains.** Modelling `||` refuses `si-fork-902`/`-952` and costs 18 `sx-union` cases @@ -312,7 +319,10 @@ None is a wrong verdict. other arity is unmodelled. - `*xpath.LookupExpr` (`?`) is modelled in `expr()` (§19.8.8.18), but the inline-function walker `bodyMentionsStreamingParam` still lacks it, along - with `*xpath.StringConcat` and `*xpath.ArgumentPlaceholder`. The + with `*xpath.StringConcat` and `*xpath.ArgumentPlaceholder`. Since v2.2.0 the + walker runs on every inline function in a streamable construct (it looks for + `current()`), so such a body is unknown anywhere, not only in a stylesheet + function. The dispatcher half of `||` is measured and reverted; see [*The `||` operator*](conformance-gaps.md#the--operator--2-cases-gained-18-lost). - `functionParamTypes` splits a declared function type by hand: the comma in @@ -1338,6 +1348,11 @@ dangerous than an open bug, and deleting one invites the same probe again. ### "An inline function declaration is grounded" — only if it mentions nothing streamed +Since v2.2.0 two more closures are unknown rather than grounded: one over a +quantified range variable bound to streamed nodes, and one calling `current()` +or `current#0`. Read literally, §19.8.8.16 grounds both; withholding the verdict +keeps execution from trusting it without refusing anything. + Three §19.8.8 expression rules were proposed with code to implement them, and reading the Recommendation first changed two of them. diff --git a/docs/stats.md b/docs/stats.md index d084fcf7..8a3a56c2 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -23,7 +23,7 @@ claim is only as good as the command behind it. | figure | count | counted by | |---|---:|---| -| Unit tests | 2,984 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | +| Unit tests | 2,986 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Fuzz targets | 12 | `grep -rn '^func Fuzz' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Limit boundary tests | 14 | `grep -hc '^func Test' ./*/limits_boundary*_test.go \| awk '{n += $1} END {print n + 0}'` | diff --git a/docs/testing.md b/docs/testing.md index a2fd5525..bbb72398 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,7 +21,7 @@ let something through, and the column that matters is the last one. | layer | count | catches | misses | |---|---:|---|---| -| **Unit tests** | 2,984 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** | 2,986 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | | **Limit boundary tests** | 14 tests | an off-by-one or an overflow at the edge of a configurable limit | a limit nobody added to the inventory | | **Race detector** | same tests | shared state a single-goroutine run never reveals | a data race on a path no test walks | | **W3C conformance suites** | 152,241 cases | systematic divergence from the specification | what the suites do not ask about — see below | diff --git a/docs/todo.md b/docs/todo.md index 6f53d934..357ac0f1 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -23,7 +23,7 @@ Current position: | XSLT 3.0 | 99.80% — 11,495 of 11,518 in scope (23 failing); 3 of those need more of the §19.8 streamability analysis | | RELAX NG | 100.00% — 965 of 965 | | Schemas wrongly refused | 7 — 6 on XSD 1.0, 1 on 1.1 | -| Tests | 2,984 `func Test` declarations, clean under `-race` | +| Tests | 2,986 `func Test` declarations, clean under `-race` | Every one of those failures, and why it is still open, is catalogued in [known-gaps.md](known-gaps.md). This file is the forward-looking half — what From 457aa0348a020292045a4fe8395c93d3334b23cf Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:39:22 -0500 Subject: [PATCH 39/45] xslt: stream a single value-of/copy-of/sequence over one record path (stage 1b(i)) A streamable xsl:source-document whose body is one xsl:value-of, xsl:copy-of or xsl:sequence reading one record path now runs in bursts: the path is evaluated per burst and the rest of the select once over all the items, with count/sum/min/max folded per burst. Bounded and pipelined consumers only; distinct-values, reverse, sort, last(), union/intersect/except and user functions stay on the tree. Any error re-runs the instruction on the tree, so errors are the tree's. --- internal/xpathleaf/host.go | 7 + xpath/leafhook.go | 6 + xslt/compile_instr.go | 11 +- xslt/instructions.go | 6 +- xslt/streamburst.go | 493 ++++++++++++++++++++++++++++++++++++- xslt/streamburst_test.go | 94 ++++++- xslt/stylesheet.go | 48 ++-- 7 files changed, 633 insertions(+), 32 deletions(-) diff --git a/internal/xpathleaf/host.go b/internal/xpathleaf/host.go index 3b08bacb..601dd9a1 100644 --- a/internal/xpathleaf/host.go +++ b/internal/xpathleaf/host.go @@ -77,3 +77,10 @@ var ( ResolveDocumentIn func(r, ctx any, uri, base string) (*xdm.Tree, error) CallItem func(ctx any, fn *xdm.FunctionItem, args []xdm.Sequence) (xdm.Sequence, error) ) + +// WithExpr returns a copy of c, an *xpath.Compiled, that evaluates e, an +// xpath.Expr parsed as c was, optimised as a compile optimises, under every +// static property c was compiled with. XSLT's burst execution evaluates a +// select expression in parts this way (xslt/streamburst.go). Package xpath +// sets it in its init. +var WithExpr func(c, e any) any diff --git a/xpath/leafhook.go b/xpath/leafhook.go index e0f6cc91..456ca95f 100644 --- a/xpath/leafhook.go +++ b/xpath/leafhook.go @@ -35,6 +35,12 @@ func init() { xpathleaf.CallItem = func(c any, fn *xdm.FunctionItem, args []xdm.Sequence) (xdm.Sequence, error) { return callItem(c.(*Context), fn, args) } + xpathleaf.WithExpr = func(c, e any) any { + n := *c.(*Compiled) + n.expr = optimize(e.(Expr)) + n.nsNeeded = readsStaticNamespaces(n.expr) + return n.withOwn() + } } // The host state with no runtime, cleared and marked: shared, since a Host is diff --git a/xslt/compile_instr.go b/xslt/compile_instr.go index 73756be5..ed1314c4 100644 --- a/xslt/compile_instr.go +++ b/xslt/compile_instr.go @@ -468,13 +468,15 @@ func (c *compiler) compileXSLInstruction(n *xdm.Node) (Instruction, error) { if err != nil { return nil, err } - return ©OfInstr{ + instr := ©OfInstr{ sel: sel, noNamespaces: noAttr(n, "copy-namespaces"), copyAccumulators: yesAttr(n, "copy-accumulators"), validation: spec, baseURI: n.BaseURI(), - }, nil + } + c.noteBurstSelect(n, instr, sel, ns) + return instr, nil case "evaluate": return c.compileEvaluate(n, ns) case "iterate": @@ -521,7 +523,9 @@ func (c *compiler) compileXSLInstruction(n *xdm.Node) (Instruction, error) { "child", ch.Name().Lexical()) } } - return &sequenceInstr{sel: sel}, nil + instr := &sequenceInstr{sel: sel} + c.noteBurstSelect(n, instr, sel, ns) + return instr, nil case "message": return c.compileMessage(n, ns) case "assert": @@ -680,6 +684,7 @@ func (c *compiler) compileValueOf(n *xdm.Node, ns xpath.NamespaceResolver) (Inst return nil, fmt.Errorf( "XTSE0870: xsl:value-of has both a select attribute and content") } + c.noteBurstSelect(n, instr, comp, ns) return instr, nil } body, err := c.compileSequence(n, n) diff --git a/xslt/instructions.go b/xslt/instructions.go index 8b54cef8..6cc10189 100644 --- a/xslt/instructions.go +++ b/xslt/instructions.go @@ -56,7 +56,7 @@ type valueOfInstr struct { func (i *valueOfInstr) Execute(rt *runtime, out *outputBuilder) error { var seq xdm.Sequence if i.sel != nil { - v, err := i.sel.Eval(rt.ctx) + v, err := rt.evalSelect(i, i.sel) if err != nil { return err } @@ -139,7 +139,7 @@ func (i *sequenceInstr) Execute(rt *runtime, out *outputBuilder) error { if i.sel == nil { return execSequence(i.body, rt, out) } - seq, err := i.sel.Eval(rt.ctx) + seq, err := rt.evalSelect(i, i.sel) if err != nil { return err } @@ -194,7 +194,7 @@ type copyOfInstr struct { } func (i *copyOfInstr) Execute(rt *runtime, out *outputBuilder) error { - seq, err := i.sel.Eval(rt.ctx) + seq, err := rt.evalSelect(i, i.sel) if err != nil { return err } diff --git a/xslt/streamburst.go b/xslt/streamburst.go index 55d968c4..8ecf95eb 100644 --- a/xslt/streamburst.go +++ b/xslt/streamburst.go @@ -22,13 +22,26 @@ package xslt // its name or attributes, may not observe node identity (is, <<, >>, union, // intersect, except, generate-id, fn:path), may not ask for last(), and may // carry nothing but atomic values from one item to the next. +// +// The body may instead be one xsl:value-of, xsl:copy-of or xsl:sequence +// whose select reads one record path, as sum(doc/rec/@v) does (stage 1b). +// The path is evaluated in each burst, and the rest of the select -- written +// around it, with the path's place taken by a variable -- once over all their +// items in order, which is what the path would have returned from the whole +// tree. count, sum, min and max of the path fold each burst's items instead +// of keeping them. The select is admitted only where its other parts read no +// focus, and the items are used only in ways a burst cannot tell from the +// tree (burstSplit). Any error is handed to the tree path, which reads the +// document again and reports it as it would have. import ( + "errors" "fmt" "io" "os" "strings" + "github.com/knroy/go-xml/v2/internal/xpathleaf" "github.com/knroy/go-xml/v2/xdm" "github.com/knroy/go-xml/v2/xpath" ) @@ -38,6 +51,9 @@ import ( var ( noBursts bool traceBursts func(href string) + // traceBurstFallback is told of each error that sends a value-of, + // copy-of or sequence target back to the tree. + traceBurstFallback func(err error) ) // burstPlan is a streamable xsl:source-document that can run in bursts. @@ -53,14 +69,38 @@ type burstPlan struct { // filtered says the last step has predicates. Without them each burst's // element is the item, and sel need not be evaluated. filtered bool + + // For a value-of, copy-of or sequence target, items is the record path + // of its select, evaluated against each burst's document node, and outer + // the select with $burstItems in the path's place. reducer, when set, + // names the count, sum, min or max of the path that folds the bursts + // instead: fold is that call over ($burstAcc, $burstItems), and folded + // the select with $burstValue in its place. + items, outer *xpath.Compiled + reducer string + fold, folded *xpath.Compiled } +// The variables the parts of a split select are bound to, in a namespace no +// stylesheet can write. +var ( + burstItems = xdm.QName{URI: "urn:x-go-xml:burst", Local: "items"} + burstAcc = xdm.QName{URI: "urn:x-go-xml:burst", Local: "acc"} + burstValue = xdm.QName{URI: "urn:x-go-xml:burst", Local: "value"} +) + +// noBurstFold makes a reducer keep every item, as any other select does; +// the tests compare the two. +var noBurstFold bool + // burstFeed is a burst plan being executed: the loop instruction it drives, // and read, which calls each with every item the loop's select would have // returned, in order. type burstFeed struct { instr Instruction read func(each func(xdm.Item) error) error + // eval is the select of a value-of, copy-of or sequence target. + eval func(rt *runtime) (xdm.Sequence, error) } // burstResolver is a document resolver that can parse a document in bursts. @@ -101,6 +141,15 @@ func planBursts(sd *xdm.Node, shared bool) *burstPlan { return nil case isXSL(kid, "for-each") || isXSL(kid, "iterate"): target = kid + case isXSL(kid, "value-of") || isXSL(kid, "copy-of") || isXSL(kid, "sequence"): + if !selectTargetOK(kid) { + return nil + } + s := splitSelect(parseAt(kid, kid.AttrValue("select"))) + if s == nil { + return nil + } + return &burstPlan{path: s.path, target: kid} case kid.Name().URI == xdm.NSXSL || !plainWrapper(kid): return nil default: @@ -127,6 +176,275 @@ func burstVerdict(sd *xdm.Node, shared bool) bool { return known && !unverified && p.streamable() && p.posture == postureGrounded } +// selectTargetOK admits a value-of, copy-of or sequence whose attributes +// change nothing bursts would have to do differently: a select, and none of +// validation, type, copy-accumulators or a separator computed from the focus. +func selectTargetOK(el *xdm.Node) bool { + if el.Attr("", "select") == nil { + return false + } + for a := range el.Attrs() { + if a.Name().URI != "" { + continue + } + switch a.Name().Local { + case "select", "copy-namespaces", "version", "xpath-default-namespace", + "default-collation", "exclude-result-prefixes", + "extension-element-prefixes", "expand-text", "use-when": + case "separator", "disable-output-escaping": + if strings.ContainsAny(a.Value(), "{}") { + return false + } + default: + return false + } + } + return true +} + +// burstSplit is a select expression taken apart around its record path. +type burstSplit struct { + // path is the element names the bursts are cut at. + path []xdm.QName + // items is the record path, put puts an expression in its place, and + // outer holds the whole expression, whatever put has made of it. + items xpath.Expr + put func(xpath.Expr) + outer *xpath.Expr + // call is the count, sum, min or max whose first argument is the + // record path, and putCall puts an expression in its place. + call *xpath.FuncCall + putCall func(xpath.Expr) +} + +// splitSelect finds the one record path in e and checks that everything +// around it either reads no focus -- it is evaluated once, with no document +// to read -- or consumes the path's items in a way that cannot tell the +// items of separate bursts from the tree's: as values, by position in the +// whole sequence, or by navigating within each item as burstSafe allows. +// Node identity, reordering and user code are refused, and so is anything +// that keeps every item when it need not (distinct-values, reverse, sort, +// last()). It returns nil when e is not admitted. +func splitSelect(e xpath.Expr) *burstSplit { + if e == nil { + return nil + } + root := e + s := &burstSplit{outer: &root} + if !s.walk(e, func(x xpath.Expr) { root = x }) || s.items == nil { + return nil + } + return s +} + +// burstReducers fold with constant state, and burstConsumers take their +// arguments whole and observe of a node no more than burstSafe allows. +var ( + burstReducers = map[string]int{"count": 1, "sum": 2, "min": 2, "max": 2} + burstConsumers = map[string]bool{ + "count": true, "sum": true, "avg": true, "min": true, "max": true, + "exists": true, "empty": true, "boolean": true, "not": true, + "head": true, "tail": true, "exactly-one": true, "zero-or-one": true, + "one-or-more": true, "deep-equal": true, "data": true, "copy-of": true, + "string": true, "number": true, "string-length": true, + "normalize-space": true, "string-join": true, "concat": true, + "contains": true, "starts-with": true, "ends-with": true, + "matches": true, "compare": true, "upper-case": true, "lower-case": true, + "substring": true, "substring-before": true, "substring-after": true, + "translate": true, "tokenize": true, "round": true, "floor": true, + "ceiling": true, "abs": true, "round-half-to-even": true, + "format-number": true, "subsequence": true, "insert-before": true, + "remove": true, "index-of": true, "name": true, "local-name": true, + "namespace-uri": true, "node-name": true, + } + burstOps = map[string]bool{ + "=": true, "!=": true, "<": true, "<=": true, ">": true, ">=": true, + "eq": true, "ne": true, "lt": true, "le": true, "gt": true, "ge": true, + "+": true, "-": true, "*": true, "div": true, "idiv": true, "mod": true, + "and": true, "or": true, + } +) + +// walk checks e, which put replaces, for splitSelect. +func (s *burstSplit) walk(e xpath.Expr, put func(xpath.Expr)) bool { + if path := recordPath(e); path != nil { + if s.items != nil { + return false // a second record path: two passes over the stream + } + s.path, s.items, s.put = path, e, put + return true + } + switch x := e.(type) { + case *xpath.FuncCall: + switch x.Name.URI { + case fnNS: + if !burstConsumers[x.Name.Local] { + return false + } + case xdm.NSXS, xdm.NSMath: + default: + return false + } + if n, ok := burstReducers[x.Name.Local]; ok && x.Name.URI == fnNS && + len(x.Args) >= 1 && len(x.Args) <= n && recordPath(x.Args[0]) != nil { + s.call, s.putCall = x, put + } + for k := range x.Args { + if !s.slot(x.Args[k], func(y xpath.Expr) { x.Args[k] = y }) { + return false + } + } + return true + case *xpath.BinaryOp: + return burstOps[x.Op] && + s.slot(x.Left, func(y xpath.Expr) { x.Left = y }) && + s.slot(x.Right, func(y xpath.Expr) { x.Right = y }) + case *xpath.UnaryOp: + return s.slot(x.Operand, func(y xpath.Expr) { x.Operand = y }) + case *xpath.CastExpr: + return s.slot(x.Operand, func(y xpath.Expr) { x.Operand = y }) + case *xpath.InstanceOfExpr: + return s.slot(x.Operand, func(y xpath.Expr) { x.Operand = y }) + case *xpath.TreatExpr: + return s.slot(x.Operand, func(y xpath.Expr) { x.Operand = y }) + case *xpath.IfExpr: + return s.slot(x.Cond, func(y xpath.Expr) { x.Cond = y }) && + s.slot(x.Then, func(y xpath.Expr) { x.Then = y }) && + s.slot(x.Else, func(y xpath.Expr) { x.Else = y }) + case *xpath.SequenceExpr: + for k := range x.Items { + if !s.slot(x.Items[k], func(y xpath.Expr) { x.Items[k] = y }) { + return false + } + } + return true + case *xpath.FilterExpr: + // The predicates see each item, positioned in the whole sequence. + return allBurstSafe(x.Predicates, false) && + s.slot(x.Base, func(y xpath.Expr) { x.Base = y }) + case *xpath.SimpleMap: + return burstSafe(x.Right, false) && + s.slot(x.Left, func(y xpath.Expr) { x.Left = y }) + case *xpath.QuantifiedExpr: + return s.bindings(x.Bindings) && burstSafe(x.Test, false) && focusFree(x.Test) + case *xpath.ForExpr: + return s.bindings(x.Bindings) && burstSafe(x.Return, false) && focusFree(x.Return) + } + return false +} + +func (s *burstSplit) bindings(bs []xpath.Binding) bool { + for k := range bs { + if !s.slot(bs[k].Seq, func(y xpath.Expr) { bs[k].Seq = y }) { + return false + } + } + return true +} + +// slot checks an operand: one that reads no focus is evaluated as it is, +// and any other must hold the record path. +func (s *burstSplit) slot(e xpath.Expr, put func(xpath.Expr)) bool { + if focusFree(e) && burstSafe(e, false) { + return true + } + return s.walk(e, put) +} + +// recordPath returns the element names to cut bursts at when e is a record +// path, nil otherwise. A record path is child steps by name from the +// document node, the last of them perhaps with predicates that do not +// depend on position (as burstPath requires), followed by steps that stay +// within the element -- down, or to its attributes, with any predicates +// burstSafe allows -- and perhaps one last expression that makes atomic +// values or copies. Each burst then holds what the path returns from one +// element, in document order, and the bursts in turn hold all of it. +func recordPath(e xpath.Expr) []xdm.QName { + var steps []xpath.Expr + switch x := e.(type) { + case *xpath.Step: + steps = []xpath.Expr{x} + case *xpath.PathExpr: + steps = x.Steps + if _, ok := firstOf(steps).(*xpath.ContextItem); ok && !x.Root { + steps = steps[1:] + } + default: + return nil + } + var path []xdm.QName + k := 0 + for ; k < len(steps); k++ { + st, ok := steps[k].(*xpath.Step) + if !ok || st.Axis != xpath.AxisChild { + break + } + nt, ok := st.Test.(*xpath.NameTest) + if !ok || nt.AnyURI || nt.AnyLocal { + break + } + if len(st.Predicates) > 0 { + // Positional predicates count within the element above, + // which is then the burst. + for _, p := range st.Predicates { + if !booleanShaped(p) || !burstSafe(p, true) { + goto tail + } + } + path = append(path, xdm.QName{URI: nt.Name.URI, Local: nt.Name.Local}) + k++ + break + } + path = append(path, xdm.QName{URI: nt.Name.URI, Local: nt.Name.Local}) + } +tail: + if len(path) == 0 { + return nil + } + for j := k; j < len(steps); j++ { + st, ok := steps[j].(*xpath.Step) + if !ok { + if j != len(steps)-1 || !tailValue(steps[j]) { + return nil + } + continue + } + switch st.Axis { + case xpath.AxisChild, xpath.AxisAttribute, xpath.AxisSelf, + xpath.AxisDescendant, xpath.AxisDescendantOrSelf: + default: + return nil + } + if !allBurstSafe(st.Predicates, false) { + return nil + } + } + return path +} + +func firstOf(es []xpath.Expr) xpath.Expr { + if len(es) == 0 { + return nil + } + return es[0] +} + +// tailValue reports whether x, the last step of a record path, makes items +// that do not depend on the tree its context node is in: atomic values, or +// copies. Its focus is each node the path has reached, so position() would +// count across the whole path. +func tailValue(x xpath.Expr) bool { + switch y := x.(type) { + case *xpath.FilterExpr: + return tailValue(y.Base) && allBurstSafe(y.Predicates, false) + case *xpath.FuncCall: + if y.Name.URI == fnNS && y.Name.Local == "copy-of" && len(y.Args) == 0 { + return true + } + } + return atomicValued(x) && burstSafe(x, true) +} + // onlyElementChild is el's one element child, when el has exactly one and // nothing else but white space. func onlyElementChild(el *xdm.Node) *xdm.Node { @@ -612,6 +930,59 @@ func (c *compiler) noteBurstLoop(n *xdm.Node, instr Instruction, sel *xpath.Comp } } +// noteBurstSelect records instr as the compiled target of the burst plan +// being compiled, when n is its value-of, copy-of or sequence, and compiles +// the parts of its select, sel, written where ns resolves names. The select +// is parsed again as compileExpr parsed it, so the parts are taken from the +// tree sel was optimised from. +func (c *compiler) noteBurstSelect(n *xdm.Node, instr Instruction, sel *xpath.Compiled, + ns xpath.NamespaceResolver) { + + p := c.burst + if p == nil || p.target != n || sel.CompatMode() { + return + } + // fn:current() is the document node, which bursts do not have. + for _, call := range sel.StaticCalls() { + if call.Name.URI == fnNS && call.Name.Local == "current" { + return + } + } + parse := func() *burstSplit { + o := exprOptions(ns) + var ( + e xpath.Expr + err error + ) + if o.RefFloor != 0 && o.RefFloor != o.Version { + e, err = xpath.ParseVersionRefFloor(sel.Source(), ns, o.Version, o.RefFloor) + } else { + e, err = xpath.ParseVersion(sel.Source(), ns, o.Version) + } + if err != nil { + return nil + } + return splitSelect(e) + } + part := func(e xpath.Expr) *xpath.Compiled { + return xpathleaf.WithExpr(sel, e).(*xpath.Compiled) + } + s := parse() + if s == nil { + return + } + s.put(&xpath.VarRef{Name: burstItems}) + p.items, p.outer = part(s.items), part(*s.outer) + if f := parse(); f != nil && f.call != nil { + f.putCall(&xpath.VarRef{Name: burstValue}) + p.reducer, p.folded = f.call.Name.Local, part(*f.outer) + args := append([]xpath.Expr{&xpath.SequenceExpr{Items: []xpath.Expr{ + &xpath.VarRef{Name: burstAcc}, &xpath.VarRef{Name: burstItems}}}}, f.call.Args[1:]...) + p.fold = part(&xpath.FuncCall{Name: f.call.Name, Args: args}) + } + p.instr, p.sel = instr, sel +} + // executeBursts runs i in bursts when its plan and its resolver allow, and // reports whether it did. func (i *sourceDocumentInstr) executeBursts(rt *runtime, out *outputBuilder, href string) (bool, error) { @@ -672,13 +1043,43 @@ func (i *sourceDocumentInstr) executeBursts(rt *runtime, out *outputBuilder, hre stand := xdm.NewTree() sub := rt.withCurrent(stand.Root, 1, 1).clearCurrentRule() plan := i.bursts - sub.bursts = &burstFeed{instr: plan.instr, read: func(each func(xdm.Item) error) error { - var bodyErr error - err := br.parseBursts(rt.ctx, href, base, plan.path, func(rec *xdm.Node) error { + // records calls fn with each burst's document node and element, + // whitespace stripped as the tree would have been. + records := func(fn func(root, rec *xdm.Node) error) error { + return br.parseBursts(rt.ctx, href, base, plan.path, func(rec *xdm.Node) error { root := rec.Tree().Root if strip != nil { root, rec = strip.stripWhitespaceForFrom(pkg, root, rec) } + return fn(root, rec) + }) + } + if plan.items != nil { + sub.bursts = &burstFeed{instr: plan.instr, eval: func(rt *runtime) (xdm.Sequence, error) { + seq, err := plan.evalBursts(rt, records) + if err == nil { + return seq, nil + } + // Whatever went wrong -- the select, a limit, the document -- + // the tree path reads the document again and fails, or not, + // exactly as it would have: an error found in a burst may not + // be the one the whole tree meets first. + if traceBurstFallback != nil { + traceBurstFallback(err) + } + unread() + root, err := i.load(rt, href) + if err != nil { + return nil, err + } + rt.streamedTrees[root] = true + return plan.sel.Eval(rt.withCurrent(root, 1, 1).ctx) + }} + return true, execSequence(i.body, sub, out) + } + sub.bursts = &burstFeed{instr: plan.instr, read: func(each func(xdm.Item) error) error { + var bodyErr error + err := records(func(root, rec *xdm.Node) error { if !plan.filtered { if err := each(rec); err != nil { bodyErr = err @@ -708,6 +1109,92 @@ func (i *sourceDocumentInstr) executeBursts(rt *runtime, out *outputBuilder, hre return true, execSequence(i.body, sub, out) } +// errBurstGiveUp ends a fold that cannot be exact; the tree path takes over. +var errBurstGiveUp = errors.New("bursts: the fold cannot be exact") + +// evalBursts evaluates the select of a value-of, copy-of or sequence target +// over the bursts records reads. The item and byte budgets are held across +// all of it, as they are across one evaluation of the select over the tree. +func (p *burstPlan) evalBursts(rt *runtime, + records func(fn func(root, rec *xdm.Node) error) error) (xdm.Sequence, error) { + + ctx := rt.ctx.HoldItemBudget().HoldByteBudget() + fold := p.reducer != "" && !noBurstFold + var ( + all, acc xdm.Sequence + n int64 + typ string + ) + err := records(func(root, _ *xdm.Node) error { + if err := ctx.Err(); err != nil { + return err + } + items, err := p.items.Eval(withFocusCurrent(ctx, root, 1, 1)) + switch { + case err != nil || len(items) == 0: + return err + case !fold: + all = append(all, items...) + case p.reducer == "count": + n += int64(len(items)) + default: + // min and max over one type compare one way throughout; over + // several, promotion makes the answer depend on the order of + // the comparisons, which only the whole sequence has. + if p.reducer != "sum" && !sameAtomicType(items, &typ) { + return errBurstGiveUp + } + acc, err = p.fold.Eval(ctx.WithVar(burstAcc, acc).WithVar(burstItems, items)) + return err + } + return nil + }) + switch { + case err != nil: + return nil, err + case !fold: + return p.outer.Eval(ctx.WithVar(burstItems, all)) + case p.reducer == "count": + acc = xdm.One(xdm.NewInteger(n)) + case acc == nil: + if acc, err = p.fold.Eval(ctx.WithVar(burstAcc, nil).WithVar(burstItems, nil)); err != nil { + return nil, err + } + } + return p.folded.Eval(ctx.WithVar(burstValue, acc)) +} + +// sameAtomicType reports whether items atomize to values of one type, the +// type of *first, which it sets from the first value seen. xs:untypedAtomic +// counts as the xs:double min and max make of it. +func sameAtomicType(items xdm.Sequence, first *string) bool { + for _, it := range xdm.Atomize(items) { + a, ok := it.(*xdm.Atomic) + if !ok { + return false + } + t := a.TypeName() + if a.Type == xdm.TypeUntypedAtomic { + t = "xs:double" + } + if *first == "" { + *first = t + } else if t != *first { + return false + } + } + return true +} + +// evalSelect evaluates sel, the select of instr, from the bursts when instr +// is the target of the plan running. +func (rt *runtime) evalSelect(instr Instruction, sel *xpath.Compiled) (xdm.Sequence, error) { + if f := rt.burstsFor(instr); f != nil && f.eval != nil { + return f.eval(rt) + } + return sel.Eval(rt.ctx) +} + // burstsFor returns the feed that drives instr, or nil when instr evaluates // its select as usual. func (rt *runtime) burstsFor(instr Instruction) *burstFeed { diff --git a/xslt/streamburst_test.go b/xslt/streamburst_test.go index 5a71ebf3..6bfd57eb 100644 --- a/xslt/streamburst_test.go +++ b/xslt/streamburst_test.go @@ -44,7 +44,7 @@ func TestBurstsMatchTree(t *testing.T) { {"identity", ``, false, ""}, {"sort", ``, false, ""}, {"node carried", ``, false, ""}, - {"sibling instruction", ``, false, ""}, + {"single instruction", ``, true, ""}, } { t.Run(tc.name, func(t *testing.T) { src := `` + @@ -216,3 +216,95 @@ func TestBurstsFailedReadIsNotRead(t *testing.T) { } } } + +// A value-of, copy-of or sequence over one record path runs in bursts when +// its consumer is bounded or pipelined, and is left to the tree otherwise. +// Each row runs with bursts, with bursts but no folding, and on the tree; all +// three must agree, error text included, and an error must have come from +// the tree path (fallback). +func TestBurstSelectsMatchTree(t *testing.T) { + const recs = `pqr` + const nan = `` + const bad = `` + const tail = `` + const ci = `'http://www.w3.org/2005/xpath-functions/collation/html-ascii-case-insensitive'` + for _, tc := range []struct { + name, body, doc string + bursts, fallback bool + }{ + // bounded reducers, folded + {"count", ``, recs, true, false}, + {"count, predicate", ``, recs, true, false}, + {"count, several per record", ``, recs, true, false}, + {"count, none", ``, recs, true, false}, + {"sum", ``, recs, true, false}, + {"sum, empty", ``, recs, true, false}, + {"sum, zero", ``, recs, true, false}, + {"sum, mixed numerics", ``, recs, true, false}, + {"sum, untyped error", ``, bad, true, true}, + {"sum, error before a malformed tail", ``, tail, true, true}, + {"max", ``, recs, true, false}, + {"min, NaN", ``, nan, true, false}, + {"max, strings by collation", ``, recs, true, false}, + {"min, strings", ``, recs, true, false}, + {"max, mixed numerics", ``, recs, true, true}, + {"max, untyped error", ``, recs, true, true}, + {"max, empty", ``, recs, true, false}, + // bounded reducers, not folded + {"avg", ``, recs, true, false}, + {"exists", ``, recs, true, false}, + {"boolean, not", ``, recs, true, false}, + {"general comparison", ``, recs, true, false}, + {"value comparison error", ``, recs, true, true}, + {"some", ``, recs, true, false}, + {"some, type error", ``, recs, true, true}, + {"literal position", ``, recs, true, false}, + {"head", ``, recs, true, false}, + {"exactly-one error", ``, recs, true, true}, + {"deep-equal", ``, recs, true, false}, + {"cast", ``, recs, true, false}, + {"positional within the record", ``, recs, true, false}, + {"positional record", ``, recs, true, false}, + // pipelined + {"simple map", ``, recs, true, false}, + {"string-join", ``, recs, true, false}, + {"copy-of path", ``, recs, true, false}, + {"copy-of filtered", ``, recs, true, false}, + {"sequence data", ``, recs, true, false}, + {"for", ``, recs, true, false}, + {"malformed tail", ``, tail, true, true}, + // refused + {"distinct-values", ``, recs, false, false}, + {"reverse", ``, recs, false, false}, + {"last()", ``, recs, false, false}, + {"union", ``, recs, false, false}, + {"two paths", ``, recs, false, false}, + {"let", ``, recs, false, false}, + {"computed separator", ``, recs, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + src := `` + + `` + + tc.body + `` + fell := false + traceBurstFallback = func(error) { fell = true } + defer func() { traceBurstFallback, noBurstFold = nil, false }() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "doc.xml"), []byte(tc.doc), 0o600); err != nil { + t.Fatal(err) + } + got, err := burstRunIn(t, src, dir, true) + ran := burstRuns + noBurstFold = true + kept, err1 := burstRunIn(t, src, dir, true) + noBurstFold = false + want, err2 := burstRunIn(t, src, dir, false) + if got != want || kept != want || fmt.Sprint(err) != fmt.Sprint(err2) || fmt.Sprint(err1) != fmt.Sprint(err2) { + t.Fatalf("bursts differ from the tree\nbursts: %q %v\nkept: %q %v\ntree: %q %v", got, err, kept, err1, want, err2) + } + if ran != tc.bursts || fell != tc.fallback { + t.Fatalf("ran in bursts: %v, fell back %v; want %v, %v (output %q, %v)", ran, fell, tc.bursts, tc.fallback, got, err) + } + }) + } +} diff --git a/xslt/stylesheet.go b/xslt/stylesheet.go index dfb10eab..c00e8ca2 100644 --- a/xslt/stylesheet.go +++ b/xslt/stylesheet.go @@ -1689,28 +1689,7 @@ func (r *nsResolver) SchemaUnionMemberTypes(name xdm.QName) ([]xdm.TypeCode, boo // for everything within. Compiling through here is what keeps that true // without threading a base URI through every call site. func compileExpr(src string, ns xpath.NamespaceResolver) (*xpath.Compiled, error) { - // The version is read from the resolver before compiling rather than - // applied after, because it decides how the expression *parses*: an - // inline function or a map constructor is a syntax error in 2.0, so - // compiling first and versioning afterwards would reject a legal 3.0 - // stylesheet before there was anything to version. - v := xpath.XPath20 - if r, ok := ns.(*nsResolver); ok { - v = r.xpathVersion - } - // A named function reference resolves at the processor's version, not the - // module's: "#N" names a function, and which functions exist is already a - // processor property (see xpath.Env.LibraryVersion). The XSLT suite - // runs version="2.0" modules scoped XSLT30+ that write "current-group#0" - // and "system-property#1", and a 3.0 processor must resolve them. Every - // other 3.0 construct stays gated on the module's own declaration. - refFloor := xpath.Version(0) - if processorAtLeast30() { - refFloor = xpath.XPath31 - } - c, err := xpath.CompileWith(src, xpath.CompileOptions{ - Namespaces: ns, Version: v, RefFloor: refFloor, - }) + c, err := xpath.CompileWith(src, exprOptions(ns)) if err != nil { return nil, err } @@ -1746,6 +1725,31 @@ func compileExpr(src string, ns xpath.NamespaceResolver) (*xpath.Compiled, error return c, nil } +// exprOptions is how compileExpr parses an expression written where ns +// resolves its names. +func exprOptions(ns xpath.NamespaceResolver) xpath.CompileOptions { + // The version is read from the resolver before compiling rather than + // applied after, because it decides how the expression *parses*: an + // inline function or a map constructor is a syntax error in 2.0, so + // compiling first and versioning afterwards would reject a legal 3.0 + // stylesheet before there was anything to version. + v := xpath.XPath20 + if r, ok := ns.(*nsResolver); ok { + v = r.xpathVersion + } + // A named function reference resolves at the processor's version, not the + // module's: "#N" names a function, and which functions exist is already a + // processor property (see xpath.Env.LibraryVersion). The XSLT suite + // runs version="2.0" modules scoped XSLT30+ that write "current-group#0" + // and "system-property#1", and a 3.0 processor must resolve them. Every + // other 3.0 construct stays gated on the module's own declaration. + refFloor := xpath.Version(0) + if processorAtLeast30() { + refFloor = xpath.XPath31 + } + return xpath.CompileOptions{Namespaces: ns, Version: v, RefFloor: refFloor} +} + func (r *nsResolver) ResolvePrefix(p string) (string, bool) { if p == "xml" { return xdm.NSXML, true From 8d3fbae1b1a341bc7220976966ca0b077a6d364c Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:42:54 -0500 Subject: [PATCH 40/45] xslt: bursts admit path steps from a split select's items, and unordered tail(doc/rec)/@v and copy-of(doc/rec)/x put their result in document order, which across bursts is the order they were made in; refused when the record path itself made copies, which bursts make earlier than the tree would. fn:unordered returns its argument unchanged here, so it pipelines. --- xslt/streamburst.go | 43 +++++++++++++++++++++++++++++++--------- xslt/streamburst_test.go | 4 ++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/xslt/streamburst.go b/xslt/streamburst.go index 8ecf95eb..9c19a9a5 100644 --- a/xslt/streamburst.go +++ b/xslt/streamburst.go @@ -254,7 +254,7 @@ var ( "translate": true, "tokenize": true, "round": true, "floor": true, "ceiling": true, "abs": true, "round-half-to-even": true, "format-number": true, "subsequence": true, "insert-before": true, - "remove": true, "index-of": true, "name": true, "local-name": true, + "remove": true, "index-of": true, "unordered": true, "name": true, "local-name": true, "namespace-uri": true, "node-name": true, } burstOps = map[string]bool{ @@ -325,6 +325,14 @@ func (s *burstSplit) walk(e xpath.Expr, put func(xpath.Expr)) bool { case *xpath.SimpleMap: return burstSafe(x.Right, false) && s.slot(x.Left, func(y xpath.Expr) { x.Left = y }) + case *xpath.PathExpr: + // Steps from the items, as a record path's own: the result is put + // in document order, which across bursts is the order they were + // made in -- the tree's, unless the path made copies of its own + // while the bursts were read, earlier than the tree would have. + return !x.Root && len(x.Steps) > 1 && pathTail(x.Steps[1:]) && + s.slot(x.Steps[0], func(y xpath.Expr) { x.Steps[0] = y }) && + (s.items == nil || !makesCopies(s.items)) case *xpath.QuantifiedExpr: return s.bindings(x.Bindings) && burstSafe(x.Test, false) && focusFree(x.Test) case *xpath.ForExpr: @@ -398,14 +406,21 @@ func recordPath(e xpath.Expr) []xdm.QName { path = append(path, xdm.QName{URI: nt.Name.URI, Local: nt.Name.Local}) } tail: - if len(path) == 0 { + if len(path) == 0 || !pathTail(steps[k:]) { return nil } - for j := k; j < len(steps); j++ { - st, ok := steps[j].(*xpath.Step) + return path +} + +// pathTail reports whether steps stay within the nodes they start from: +// down or to attributes, with any predicates burstSafe allows, and perhaps +// one last expression that makes values (tailValue). +func pathTail(steps []xpath.Expr) bool { + for j, s := range steps { + st, ok := s.(*xpath.Step) if !ok { - if j != len(steps)-1 || !tailValue(steps[j]) { - return nil + if j != len(steps)-1 || !tailValue(s) { + return false } continue } @@ -413,13 +428,23 @@ tail: case xpath.AxisChild, xpath.AxisAttribute, xpath.AxisSelf, xpath.AxisDescendant, xpath.AxisDescendantOrSelf: default: - return nil + return false } if !allBurstSafe(st.Predicates, false) { - return nil + return false } } - return path + return true +} + +// makesCopies reports whether the record path p ends in copy-of(). +func makesCopies(p xpath.Expr) bool { + pe, ok := p.(*xpath.PathExpr) + if !ok { + return false + } + f, ok := firstOf(pe.Steps[len(pe.Steps)-1:]).(*xpath.FuncCall) + return ok && f.Name.URI == fnNS && f.Name.Local == "copy-of" } func firstOf(es []xpath.Expr) xpath.Expr { diff --git a/xslt/streamburst_test.go b/xslt/streamburst_test.go index 6bfd57eb..3b3e2763 100644 --- a/xslt/streamburst_test.go +++ b/xslt/streamburst_test.go @@ -272,6 +272,9 @@ func TestBurstSelectsMatchTree(t *testing.T) { {"copy-of filtered", ``, recs, true, false}, {"sequence data", ``, recs, true, false}, {"for", ``, recs, true, false}, + {"path from the items", ``, recs, true, false}, + {"path from copies", ``, recs, true, false}, + {"unordered", ``, recs, true, false}, {"malformed tail", ``, tail, true, true}, // refused {"distinct-values", ``, recs, false, false}, @@ -279,6 +282,7 @@ func TestBurstSelectsMatchTree(t *testing.T) { {"last()", ``, recs, false, false}, {"union", ``, recs, false, false}, {"two paths", ``, recs, false, false}, + {"path from copies the path made", ``, recs, false, false}, {"let", ``, recs, false, false}, {"computed separator", ``, recs, false, false}, } { From 9f3d8151caa3dd74ffc89c93055b1e6afba4f0c3 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 17:53:42 -0500 Subject: [PATCH 41/45] xslt: atomize a split select's items per burst when its consumer atomizes them A record path kept as nodes holds each burst's tree, about 950 bytes a record against 208 in the tree: avg(doc/rec/@v) over 100 MB peaked at 3.8 GB heap against 1.3 GB on the tree. Where the path's own consumer atomizes it (aggregates, string functions, comparisons, arithmetic, casts) the items are atomized as each burst is read: 609 MB. --- xslt/streamburst.go | 41 +++++++++++++++++++++++++++++++++++- xslt/streamburst_test.go | 45 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/xslt/streamburst.go b/xslt/streamburst.go index 9c19a9a5..e0fdb541 100644 --- a/xslt/streamburst.go +++ b/xslt/streamburst.go @@ -79,6 +79,9 @@ type burstPlan struct { items, outer *xpath.Compiled reducer string fold, folded *xpath.Compiled + // atomize says the path's items are consumed only as atomic values, + // so each burst's are atomized at once and its tree let go. + atomize bool } // The variables the parts of a split select are bound to, in a namespace no @@ -215,6 +218,9 @@ type burstSplit struct { // record path, and putCall puts an expression in its place. call *xpath.FuncCall putCall func(xpath.Expr) + // atomize says the expression consuming the record path atomizes it + // and keeps none of its nodes (burstAtomizers), or only counts them. + atomize bool } // splitSelect finds the one record path in e and checks that everything @@ -257,6 +263,20 @@ var ( "remove": true, "index-of": true, "unordered": true, "name": true, "local-name": true, "namespace-uri": true, "node-name": true, } + // burstAtomizers turn every node of their arguments into its typed + // value before using it (count, exists and empty use one item per + // node, which an atomized untyped node still is), and return no node. + burstAtomizers = map[string]bool{ + "count": true, "exists": true, "empty": true, "sum": true, "avg": true, + "min": true, "max": true, "data": true, "string": true, "number": true, + "string-length": true, "normalize-space": true, "string-join": true, + "concat": true, "contains": true, "starts-with": true, "ends-with": true, + "matches": true, "compare": true, "upper-case": true, "lower-case": true, + "substring": true, "substring-before": true, "substring-after": true, + "translate": true, "tokenize": true, "round": true, "floor": true, + "ceiling": true, "abs": true, "round-half-to-even": true, + "format-number": true, "index-of": true, + } burstOps = map[string]bool{ "=": true, "!=": true, "<": true, "<=": true, ">": true, ">=": true, "eq": true, "ne": true, "lt": true, "le": true, "gt": true, "ge": true, @@ -289,6 +309,9 @@ func (s *burstSplit) walk(e xpath.Expr, put func(xpath.Expr)) bool { len(x.Args) >= 1 && len(x.Args) <= n && recordPath(x.Args[0]) != nil { s.call, s.putCall = x, put } + if x.Name.URI == xdm.NSXS || x.Name.URI == fnNS && burstAtomizers[x.Name.Local] { + s.atomizes(x.Args...) + } for k := range x.Args { if !s.slot(x.Args[k], func(y xpath.Expr) { x.Args[k] = y }) { return false @@ -296,12 +319,16 @@ func (s *burstSplit) walk(e xpath.Expr, put func(xpath.Expr)) bool { } return true case *xpath.BinaryOp: + if x.Op != "and" && x.Op != "or" { + s.atomizes(x.Left, x.Right) + } return burstOps[x.Op] && s.slot(x.Left, func(y xpath.Expr) { x.Left = y }) && s.slot(x.Right, func(y xpath.Expr) { x.Right = y }) case *xpath.UnaryOp: return s.slot(x.Operand, func(y xpath.Expr) { x.Operand = y }) case *xpath.CastExpr: + s.atomizes(x.Operand) return s.slot(x.Operand, func(y xpath.Expr) { x.Operand = y }) case *xpath.InstanceOfExpr: return s.slot(x.Operand, func(y xpath.Expr) { x.Operand = y }) @@ -341,6 +368,16 @@ func (s *burstSplit) walk(e xpath.Expr, put func(xpath.Expr)) bool { return false } +// atomizes notes that the operands es, of an expression that atomizes them, +// include the record path itself. +func (s *burstSplit) atomizes(es ...xpath.Expr) { + for _, e := range es { + if recordPath(e) != nil { + s.atomize = true + } + } +} + func (s *burstSplit) bindings(bs []xpath.Binding) bool { for k := range bs { if !s.slot(bs[k].Seq, func(y xpath.Expr) { bs[k].Seq = y }) { @@ -997,7 +1034,7 @@ func (c *compiler) noteBurstSelect(n *xdm.Node, instr Instruction, sel *xpath.Co return } s.put(&xpath.VarRef{Name: burstItems}) - p.items, p.outer = part(s.items), part(*s.outer) + p.items, p.outer, p.atomize = part(s.items), part(*s.outer), s.atomize if f := parse(); f != nil && f.call != nil { f.putCall(&xpath.VarRef{Name: burstValue}) p.reducer, p.folded = f.call.Name.Local, part(*f.outer) @@ -1158,6 +1195,8 @@ func (p *burstPlan) evalBursts(rt *runtime, switch { case err != nil || len(items) == 0: return err + case !fold && p.atomize: + all = append(all, xdm.Atomize(items)...) case !fold: all = append(all, items...) case p.reducer == "count": diff --git a/xslt/streamburst_test.go b/xslt/streamburst_test.go index 3b3e2763..3c87fd88 100644 --- a/xslt/streamburst_test.go +++ b/xslt/streamburst_test.go @@ -5,8 +5,11 @@ import ( "fmt" "os" "path/filepath" + goruntime "runtime" "strings" + "sync/atomic" "testing" + "time" "github.com/knroy/go-xml/v2/xdm" ) @@ -312,3 +315,45 @@ func TestBurstSelectsMatchTree(t *testing.T) { }) } } + +// A select that atomizes its record path keeps the values, not each burst's +// tree: avg over 40,000 records stays within a few megabytes, where holding +// the attribute nodes kept every burst (about 950 bytes a record, 38 MB). +func TestBurstSelectAtomizes(t *testing.T) { + var b strings.Builder + b.WriteString("") + for i := range 40000 { + fmt.Fprintf(&b, "t\n", i, i%97) + } + b.WriteString("") + src := ` + + ` + goruntime.GC() + var m0 goruntime.MemStats + goruntime.ReadMemStats(&m0) + var peak atomic.Uint64 + stop, done := make(chan struct{}), make(chan struct{}) + go func() { + defer close(done) + var m goruntime.MemStats + for { + goruntime.ReadMemStats(&m) + if m.HeapAlloc > peak.Load() { + peak.Store(m.HeapAlloc) + } + select { + case <-stop: + return + case <-time.After(time.Millisecond): + } + } + }() + out, err := burstRun(t, src, "big.xml", b.String(), true) + close(stop) + <-done + grew := (int64(peak.Load()) - int64(m0.HeapAlloc)) >> 20 + if err != nil || !strings.HasSuffix(out, "47.97255") || !burstRuns || grew > 20 { + t.Fatalf("got %q, %v, bursts %v, heap grew %d MB", out, err, burstRuns, grew) + } +} From 48b1a38cd51efbb7b58687822f3fb5fcda83bec4 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 18:07:06 -0500 Subject: [PATCH 42/45] xslt: refuse a split select that would keep the record path's nodes A burst node holds its burst's whole tree, 945 bytes a record against 208 in the document's tree, so copy-of(doc/rec), filters, simple maps and head/tail over the path's nodes used more memory streamed than on the tree. The gate now admits a select only where the path's items are folded, atomized by their consumer (or by xsl:value-of of the bare path), or are values or copies already; the path steps from items this made unreachable are removed. --- xslt/streamburst.go | 54 ++++++++++++++++++++++++---------------- xslt/streamburst_test.go | 35 ++++++++++++++++---------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/xslt/streamburst.go b/xslt/streamburst.go index e0fdb541..da6fdfd4 100644 --- a/xslt/streamburst.go +++ b/xslt/streamburst.go @@ -29,7 +29,9 @@ package xslt // around it, with the path's place taken by a variable -- once over all their // items in order, which is what the path would have returned from the whole // tree. count, sum, min and max of the path fold each burst's items instead -// of keeping them. The select is admitted only where its other parts read no +// of keeping them. Any other select must atomize the items as each burst is read, or +// have values or copies for items: one that would keep a burst's nodes is +// left to the tree, since each node keeps its burst's whole tree. The select is admitted only where its other parts read no // focus, and the items are used only in ways a burst cannot tell from the // tree (burstSplit). Any error is handed to the tree path, which reads the // document again and reports it as it would have. @@ -148,7 +150,7 @@ func planBursts(sd *xdm.Node, shared bool) *burstPlan { if !selectTargetOK(kid) { return nil } - s := splitSelect(parseAt(kid, kid.AttrValue("select"))) + s := splitSelect(parseAt(kid, kid.AttrValue("select")), isXSL(kid, "value-of")) if s == nil { return nil } @@ -231,7 +233,16 @@ type burstSplit struct { // Node identity, reordering and user code are refused, and so is anything // that keeps every item when it need not (distinct-values, reverse, sort, // last()). It returns nil when e is not admitted. -func splitSelect(e xpath.Expr) *burstSplit { +// +// It also refuses a select that would keep the record path's nodes until +// the end: a node of a burst holds the burst's whole tree, several times +// what the same element costs in the document's tree, so keeping them +// would make streaming dearer than the tree it replaces. The path's items +// must be folded (count, sum, min, max), atomized by the expression that +// consumes them (burstAtomizers; xsl:value-of, valueOf, when the path is +// its whole select), or values or copies already (a last step tailValue +// admits). +func splitSelect(e xpath.Expr, valueOf bool) *burstSplit { if e == nil { return nil } @@ -240,9 +251,26 @@ func splitSelect(e xpath.Expr) *burstSplit { if !s.walk(e, func(x xpath.Expr) { root = x }) || s.items == nil { return nil } + if valueOf && root == s.items { + s.atomize = true + } + if s.call == nil && !s.atomize && !endsInValue(s.items) { + return nil + } return s } +// endsInValue reports whether the record path p ends in an expression +// rather than a step: one that makes values or copies (tailValue). +func endsInValue(p xpath.Expr) bool { + pe, ok := p.(*xpath.PathExpr) + if !ok { + return false + } + _, step := pe.Steps[len(pe.Steps)-1].(*xpath.Step) + return !step +} + // burstReducers fold with constant state, and burstConsumers take their // arguments whole and observe of a node no more than burstSafe allows. var ( @@ -352,14 +380,6 @@ func (s *burstSplit) walk(e xpath.Expr, put func(xpath.Expr)) bool { case *xpath.SimpleMap: return burstSafe(x.Right, false) && s.slot(x.Left, func(y xpath.Expr) { x.Left = y }) - case *xpath.PathExpr: - // Steps from the items, as a record path's own: the result is put - // in document order, which across bursts is the order they were - // made in -- the tree's, unless the path made copies of its own - // while the bursts were read, earlier than the tree would have. - return !x.Root && len(x.Steps) > 1 && pathTail(x.Steps[1:]) && - s.slot(x.Steps[0], func(y xpath.Expr) { x.Steps[0] = y }) && - (s.items == nil || !makesCopies(s.items)) case *xpath.QuantifiedExpr: return s.bindings(x.Bindings) && burstSafe(x.Test, false) && focusFree(x.Test) case *xpath.ForExpr: @@ -474,16 +494,6 @@ func pathTail(steps []xpath.Expr) bool { return true } -// makesCopies reports whether the record path p ends in copy-of(). -func makesCopies(p xpath.Expr) bool { - pe, ok := p.(*xpath.PathExpr) - if !ok { - return false - } - f, ok := firstOf(pe.Steps[len(pe.Steps)-1:]).(*xpath.FuncCall) - return ok && f.Name.URI == fnNS && f.Name.Local == "copy-of" -} - func firstOf(es []xpath.Expr) xpath.Expr { if len(es) == 0 { return nil @@ -1024,7 +1034,7 @@ func (c *compiler) noteBurstSelect(n *xdm.Node, instr Instruction, sel *xpath.Co if err != nil { return nil } - return splitSelect(e) + return splitSelect(e, isXSL(n, "value-of")) } part := func(e xpath.Expr) *xpath.Compiled { return xpathleaf.WithExpr(sel, e).(*xpath.Compiled) diff --git a/xslt/streamburst_test.go b/xslt/streamburst_test.go index 3c87fd88..516f72e7 100644 --- a/xslt/streamburst_test.go +++ b/xslt/streamburst_test.go @@ -256,30 +256,39 @@ func TestBurstSelectsMatchTree(t *testing.T) { // bounded reducers, not folded {"avg", ``, recs, true, false}, {"exists", ``, recs, true, false}, - {"boolean, not", ``, recs, true, false}, + {"boolean, not", ``, recs, false, false}, {"general comparison", ``, recs, true, false}, {"value comparison error", ``, recs, true, true}, - {"some", ``, recs, true, false}, - {"some, type error", ``, recs, true, true}, - {"literal position", ``, recs, true, false}, + {"some", ``, recs, false, false}, + {"some, type error", ``, recs, false, false}, + {"literal position", ``, recs, false, false}, {"head", ``, recs, true, false}, - {"exactly-one error", ``, recs, true, true}, + {"exactly-one error", ``, recs, false, false}, {"deep-equal", ``, recs, true, false}, {"cast", ``, recs, true, false}, {"positional within the record", ``, recs, true, false}, - {"positional record", ``, recs, true, false}, + {"positional record", ``, recs, false, false}, // pipelined - {"simple map", ``, recs, true, false}, + {"simple map", ``, recs, false, false}, {"string-join", ``, recs, true, false}, - {"copy-of path", ``, recs, true, false}, - {"copy-of filtered", ``, recs, true, false}, + {"copy-of path", ``, recs, false, false}, + {"copy-of filtered", ``, recs, false, false}, {"sequence data", ``, recs, true, false}, {"for", ``, recs, true, false}, - {"path from the items", ``, recs, true, false}, - {"path from copies", ``, recs, true, false}, - {"unordered", ``, recs, true, false}, + {"path from the items", ``, recs, false, false}, + {"path from copies", ``, recs, false, false}, + {"unordered", ``, recs, false, false}, {"malformed tail", ``, tail, true, true}, - // refused + // the same consumers over values: admitted + {"boolean over values", ``, recs, true, true}, + {"some over values", ``, recs, true, false}, + {"literal position over values", ``, recs, true, false}, + {"exactly-one over values", ``, recs, true, true}, + {"simple map over values", ``, recs, true, false}, + {"copies the path made", ``, recs, true, false}, + {"unordered over values", ``, recs, true, false}, + {"value-of of the path", ``, recs, true, false}, + // refused (the rows above that expect no bursts keep the path's nodes) {"distinct-values", ``, recs, false, false}, {"reverse", ``, recs, false, false}, {"last()", ``, recs, false, false}, From 3157566de2a004678f54296780f5de5573fe597c Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 18:09:59 -0500 Subject: [PATCH 43/45] xslt: refuse a split select that would keep the copies its record path made doc/rec/copy-of() kept to the end is one tree per record: at 100 MB it peaked at 4.2 GB RSS streamed against 4.0 GB on the tree. Such copies are admitted only where they are folded or atomized. --- xslt/streamburst.go | 31 +++++++++++++++++++++---------- xslt/streamburst_test.go | 3 ++- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/xslt/streamburst.go b/xslt/streamburst.go index da6fdfd4..e1925265 100644 --- a/xslt/streamburst.go +++ b/xslt/streamburst.go @@ -29,11 +29,12 @@ package xslt // around it, with the path's place taken by a variable -- once over all their // items in order, which is what the path would have returned from the whole // tree. count, sum, min and max of the path fold each burst's items instead -// of keeping them. Any other select must atomize the items as each burst is read, or -// have values or copies for items: one that would keep a burst's nodes is -// left to the tree, since each node keeps its burst's whole tree. The select is admitted only where its other parts read no -// focus, and the items are used only in ways a burst cannot tell from the -// tree (burstSplit). Any error is handed to the tree path, which reads the +// of keeping them; any other select must atomize the items as each burst is +// read, or have atomic values for items. One that would keep a burst's +// nodes, or copies of them, is left to the tree, which then costs no more. +// The select is admitted only where its other parts read no focus, and the +// items are used only in ways a burst cannot tell from the tree +// (burstSplit). Any error is handed to the tree path, which reads the // document again and reports it as it would have. import ( @@ -240,8 +241,9 @@ type burstSplit struct { // would make streaming dearer than the tree it replaces. The path's items // must be folded (count, sum, min, max), atomized by the expression that // consumes them (burstAtomizers; xsl:value-of, valueOf, when the path is -// its whole select), or values or copies already (a last step tailValue -// admits). +// its whole select), or atomic values already. Copies made by a last +// copy-of() are refused as well: kept to the end, each one a tree of its +// own, they peaked above the tree path's RSS (4.2 GB against 4.0 GB, 100 MB). func splitSelect(e xpath.Expr, valueOf bool) *burstSplit { if e == nil { return nil @@ -261,14 +263,23 @@ func splitSelect(e xpath.Expr, valueOf bool) *burstSplit { } // endsInValue reports whether the record path p ends in an expression -// rather than a step: one that makes values or copies (tailValue). +// that makes atomic values: one tailValue admits other than copy-of(). func endsInValue(p xpath.Expr) bool { pe, ok := p.(*xpath.PathExpr) if !ok { return false } - _, step := pe.Steps[len(pe.Steps)-1].(*xpath.Step) - return !step + last := pe.Steps[len(pe.Steps)-1] + if f, ok := last.(*xpath.FilterExpr); ok { + last = f.Base + } + switch x := last.(type) { + case *xpath.Step: + return false + case *xpath.FuncCall: + return x.Name.URI != fnNS || x.Name.Local != "copy-of" + } + return true } // burstReducers fold with constant state, and burstConsumers take their diff --git a/xslt/streamburst_test.go b/xslt/streamburst_test.go index 516f72e7..4b2df453 100644 --- a/xslt/streamburst_test.go +++ b/xslt/streamburst_test.go @@ -285,7 +285,8 @@ func TestBurstSelectsMatchTree(t *testing.T) { {"literal position over values", ``, recs, true, false}, {"exactly-one over values", ``, recs, true, true}, {"simple map over values", ``, recs, true, false}, - {"copies the path made", ``, recs, true, false}, + {"copies the path made", ``, recs, false, false}, + {"copies the path made, counted", ``, recs, true, false}, {"unordered over values", ``, recs, true, false}, {"value-of of the path", ``, recs, true, false}, // refused (the rows above that expect no bursts keep the path's nodes) From 49445112d60e1c8f13de15db9663d62933771ef8 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 18:15:03 -0500 Subject: [PATCH 44/45] xslt: atomize a bare value-of select per burst only when it ends in attributes xsl:value-of joins adjacent text nodes without its separator, so atomizing doc/rec/x/text() per burst put a space between records (si-value-of-009). Attributes are as good as their values to it; other node paths stay on the tree. --- xslt/streamburst.go | 17 +++++++++++++++-- xslt/streamburst_test.go | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/xslt/streamburst.go b/xslt/streamburst.go index e1925265..76c6ed46 100644 --- a/xslt/streamburst.go +++ b/xslt/streamburst.go @@ -241,7 +241,7 @@ type burstSplit struct { // would make streaming dearer than the tree it replaces. The path's items // must be folded (count, sum, min, max), atomized by the expression that // consumes them (burstAtomizers; xsl:value-of, valueOf, when the path is -// its whole select), or atomic values already. Copies made by a last +// its whole select and ends in attributes), or atomic values already. Copies made by a last // copy-of() are refused as well: kept to the end, each one a tree of its // own, they peaked above the tree path's RSS (4.2 GB against 4.0 GB, 100 MB). func splitSelect(e xpath.Expr, valueOf bool) *burstSplit { @@ -253,7 +253,9 @@ func splitSelect(e xpath.Expr, valueOf bool) *burstSplit { if !s.walk(e, func(x xpath.Expr) { root = x }) || s.items == nil { return nil } - if valueOf && root == s.items { + // xsl:value-of joins adjacent text nodes with no separator, so only + // attributes are as good as their values to it. + if valueOf && root == s.items && endsInAttribute(s.items) { s.atomize = true } if s.call == nil && !s.atomize && !endsInValue(s.items) { @@ -262,6 +264,17 @@ func splitSelect(e xpath.Expr, valueOf bool) *burstSplit { return s } +// endsInAttribute reports whether the record path p ends in a step on the +// attribute axis. +func endsInAttribute(p xpath.Expr) bool { + var last xpath.Expr = p + if pe, ok := p.(*xpath.PathExpr); ok { + last = pe.Steps[len(pe.Steps)-1] + } + st, ok := last.(*xpath.Step) + return ok && st.Axis == xpath.AxisAttribute +} + // endsInValue reports whether the record path p ends in an expression // that makes atomic values: one tailValue admits other than copy-of(). func endsInValue(p xpath.Expr) bool { diff --git a/xslt/streamburst_test.go b/xslt/streamburst_test.go index 4b2df453..4add77b2 100644 --- a/xslt/streamburst_test.go +++ b/xslt/streamburst_test.go @@ -289,6 +289,7 @@ func TestBurstSelectsMatchTree(t *testing.T) { {"copies the path made, counted", ``, recs, true, false}, {"unordered over values", ``, recs, true, false}, {"value-of of the path", ``, recs, true, false}, + {"value-of of text nodes", ``, recs, false, false}, // refused (the rows above that expect no bursts keep the path's nodes) {"distinct-values", ``, recs, false, false}, {"reverse", ``, recs, false, false}, From 1337974f1892485145f5e02b994e6dca93250b82 Mon Sep 17 00:00:00 2001 From: Komol Nath Roy Date: Sun, 1 Nov 2026 18:32:00 -0500 Subject: [PATCH 45/45] =?UTF-8?q?docs:=20streaming=20stage=201b(i)=20?= =?UTF-8?q?=E2=80=94=20aggregates=20over=20one=20record=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 6 ++++++ README.md | 10 ++++++---- docs/conformance-gaps.md | 2 +- docs/known-gaps.md | 33 +++++++++++++++++++++++++++------ docs/stats.md | 2 +- docs/testing.md | 2 +- docs/todo.md | 2 +- 7 files changed, 43 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b3a6c5a..1b2d03b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ is byte-identical; new API is additive. | Change | What it does | Commit | |---|---|---| +| Streamed aggregates over records | A lone `value-of`/`copy-of`/`sequence` over one record path streams too; `count`/`sum`/`min`/`max` fold per record: 1 GB summed in 33 MB RSS. Any error re-runs on the tree. | [`457aa03`][457aa03], [`9f3d815`][9f3d815], [`48b1a38`][48b1a38], [`3157566`][3157566], [`4944511`][4944511] | | Streamed `xsl:source-document`, first stage | A streamable `xsl:for-each`/`xsl:iterate` over a child path reads one record at a time: 1 GB totalled at 27 MB RSS, not 3.7 GB, at about 2× CPU. Other shapes build the tree. | [`3f185fe`][3f185fe], [`27757b3`][27757b3] | | `xdm.ParseBursts` | Parses a document handing over one selected element at a time in a tree of its own, with limits counted over the whole document; memory stays at the spine plus one record. | [`b4b6052`][b4b6052], [`80f999f`][80f999f] | | `xslt.FileResolver.MaxNodes`; `MaxBytes` reaches the parser | Raising `MaxBytes` past 64 MB had no effect on `fn:doc`, the parser kept its own default. Both limits now apply to what the resolver parses. | [`80f999f`][80f999f] | @@ -1884,3 +1885,8 @@ here so every entry in this file sits under a release. [7920635]: https://github.com/knroy/go-xml/commit/7920635 [b12ecbf]: https://github.com/knroy/go-xml/commit/b12ecbf [e94cdf9]: https://github.com/knroy/go-xml/commit/e94cdf9 +[457aa03]: https://github.com/knroy/go-xml/commit/457aa03 +[9f3d815]: https://github.com/knroy/go-xml/commit/9f3d815 +[48b1a38]: https://github.com/knroy/go-xml/commit/48b1a38 +[3157566]: https://github.com/knroy/go-xml/commit/3157566 +[4944511]: https://github.com/knroy/go-xml/commit/4944511 diff --git a/README.md b/README.md index 447dca25..bd38a2b0 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ and maintains it as a project of his own. | **XSD 1.1** | 99.98% instance (26,217 of 26,222); **99.97%** schema-validity (15,350 of 15,354); opt-in via `Version11` | | **RELAX NG** | 100.00% of James Clark's spectest (965 of 965 assertions); XML and compact syntax, from Go and from `go-xml validate -rng` | | **DTD** | content models, attribute defaults, enumerations, `ID`/`IDREF`; external subset, parameter entities across both subsets, conditional sections — via `dtd.Load` with a caller-supplied resolver, nothing fetched by default | -| **Tests** | 2,986 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | +| **Tests** | 2,988 `func Test` declarations, clean under `-race` (a few subtests skip without the corpora below) | | **Production schemas** | UBL 2.1, UN/CEFACT CII, Factur-X/ZUGFeRD, Peppol BIS 3.0 — 88 schemas load, instances validate clean | | **API** | 1.2; the exported surface is stable and additive over 1.1, and a breaking change means 2.0 with a new module path | @@ -178,9 +178,11 @@ true here: each test's declared version dependency — a different kind of measurement from running a suite written for the version under test. - **Streamed execution is implemented only for the narrowest shape** (an + **Streamed execution is implemented only for two narrow shapes** (an `xsl:source-document streamable="yes"` whose body is one `xsl:for-each` or - `xsl:iterate` over a child path, read one record at a time; see + `xsl:iterate` over a child path, or one `xsl:value-of`/`xsl:copy-of`/ + `xsl:sequence` over one record path such as `sum(doc/rec/@v)`, read one + record at a time; see [known gaps](docs/known-gaps.md)). Otherwise the streaming cases are in scope and counted: the stylesheet-level half of streaming, the §19.8 streamability analysis that refuses a construct with `XTSE3430`, is @@ -1391,7 +1393,7 @@ back, is in [docs/testing.md](docs/testing.md). | method | what it catches | what it misses | |---|---|---| -| **Unit tests** (2,986 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** (2,988 `func Test` declarations) | places where a plausible implementation is quietly wrong | anything nobody thought to write a test for | | **Spec inventories** | features absent entirely | features present but behaving wrongly | | **Saxon differential** | subtle behavioural divergence on real stylesheets | constructs the corpora do not use | | **W3C QT3 suite** | systematic conformance across 22,054 XPath and 30,517 XQuery cases | XSLT (it is an XPath suite) | diff --git a/docs/conformance-gaps.md b/docs/conformance-gaps.md index 3bc71a60..62814bd0 100644 --- a/docs/conformance-gaps.md +++ b/docs/conformance-gaps.md @@ -49,7 +49,7 @@ W3C disagreements: 0 + 0 + 0 + 1 + 8 + 23 + 30 + 31 + 0 = 93. Measured 2026-09-1 -The unit-test suite is 2,986 tests. +The unit-test suite is 2,988 tests. The last two rows are not W3C suites but real-world corpora — DocBook xslTNG's diff --git a/docs/known-gaps.md b/docs/known-gaps.md index 85a50409..d45ddc5a 100644 --- a/docs/known-gaps.md +++ b/docs/known-gaps.md @@ -177,7 +177,7 @@ Measured on branches in 2026-11 and not adopted, each with the reason: Namespace nodes are held weakly since v2.2.0; while a tree's table is not empty, every GC cycle sweeps it. -### Streamed execution covers one shape (XSLT 3.0, performance) +### Streamed execution covers two shapes (XSLT 3.0, performance) Since v2.2.0 an `xsl:source-document streamable="yes"` whose body is one `xsl:for-each` or `xsl:iterate` (optionally inside constant literal result @@ -186,13 +186,34 @@ the last step only, and a body the gate recognises with a known and verified §19.8 verdict, runs record by record through `xdm.ParseBursts`: a 1 GB file totalled by `xsl:iterate` peaks at 27 MB RSS against 3.7 GB, at about 2× the CPU. Everything else builds the tree, -as before. In the W3C suite 9 of the 2,599 streamable source documents take the -burst path; the commonest refusals are a body of `xsl:value-of` (840), -`xsl:copy-of` (344) or `xsl:apply-templates` (175). Also: +as before. + +The second shape is a body that is one `xsl:value-of`, `xsl:copy-of` or +`xsl:sequence` (again optionally inside constant literal result elements) whose +select reads one record path, such as `sum(doc/rec/@v)` or +`count(doc/rec[@k = 'x'])`. The path runs per record; the rest of the select +runs once over the values. `count`, `sum`, `min` and `max` fold per record and +stay flat: over a 1 GB file of 25.9M records, `count` peaks at 3.4 MB heap / +25 MB RSS and `sum` at 11 MB / 33 MB, where tree mode needs 773–1,284 MB at +100 MB. Other consumers (`avg`, comparisons, `string-join`, a bare `value-of` +of attributes) keep one atomic value per item, about half the tree's memory, +still growing with the input. A select that would keep the records' nodes or +copies (`copy-of(doc/rec)`, filters returning nodes, `head`/`tail` passed on) +is refused and runs on the tree, because a kept record costs more there than in +a whole tree (945 B against 208 B). Also refused: `distinct-values`, `reverse`, +`sort`, `last()`, union/intersect/except, `let`, user functions, wildcard steps. + +In the W3C XSLT 3.0 run 364 cases take the burst path (9 before the second +shape). Of the 786 single-instruction bodies over one record path with a known +and verified verdict, 294 are admitted (261 bounded reducers, 33 pipelined). +Also: - **The result is still built in memory**, so a large filtered copy grows with its output and stops at the result-node cap. -- **Error order can differ from tree mode.** Tree mode reads the whole file, +- **Error order can differ from tree mode, for the loop shape.** (The single-instruction + shape re-runs on the tree after any error, so its errors are the tree's; the + cost is a second read of the document, and the 1 GiB string budget charges the + strings both attempts built.) Tree mode reads the whole file, then evaluates the whole `select`, before the body runs; bursts do each per record. So a dynamic error in the body is raised before a malformed tail, a `MaxNodes`/`MaxBytes` refusal or an encoding error later in the file, and @@ -237,7 +258,7 @@ and §18.1's grounded demand at both the sites it names. All three fail in the same direction: the suite expects `XTSE3430` — *this construct is not guaranteed streamable* — and the transform succeeds instead. The engine builds a tree for these (it reads in bursts only the narrow shape -described under *Streamed execution covers one shape*), so it produces the **right +described under *Streamed execution covers two shapes*), so it produces the **right answer** for all three; what it does not produce is the static refusal §19.8 requires a streaming processor to make before running anything. These are not wrong answers; they are a static analysis that is not complete. diff --git a/docs/stats.md b/docs/stats.md index 8a3a56c2..b6f5c8a7 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -23,7 +23,7 @@ claim is only as good as the command behind it. | figure | count | counted by | |---|---:|---| -| Unit tests | 2,986 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | +| Unit tests | 2,988 | `grep -rn '^func Test' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Fuzz targets | 12 | `grep -rn '^func Fuzz' --include='*_test.go' . \| grep -v '/\.claude/worktrees/' \| grep -vc '^\./bench/'` | | Limit boundary tests | 14 | `grep -hc '^func Test' ./*/limits_boundary*_test.go \| awk '{n += $1} END {print n + 0}'` | diff --git a/docs/testing.md b/docs/testing.md index bbb72398..92a5ff35 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,7 +21,7 @@ let something through, and the column that matters is the last one. | layer | count | catches | misses | |---|---:|---|---| -| **Unit tests** | 2,986 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | +| **Unit tests** | 2,988 | a plausible implementation that is quietly wrong | anything nobody thought to write a test for | | **Limit boundary tests** | 14 tests | an off-by-one or an overflow at the edge of a configurable limit | a limit nobody added to the inventory | | **Race detector** | same tests | shared state a single-goroutine run never reveals | a data race on a path no test walks | | **W3C conformance suites** | 152,241 cases | systematic divergence from the specification | what the suites do not ask about — see below | diff --git a/docs/todo.md b/docs/todo.md index 357ac0f1..af9334dc 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -23,7 +23,7 @@ Current position: | XSLT 3.0 | 99.80% — 11,495 of 11,518 in scope (23 failing); 3 of those need more of the §19.8 streamability analysis | | RELAX NG | 100.00% — 965 of 965 | | Schemas wrongly refused | 7 — 6 on XSD 1.0, 1 on 1.1 | -| Tests | 2,986 `func Test` declarations, clean under `-race` | +| Tests | 2,988 `func Test` declarations, clean under `-race` | Every one of those failures, and why it is still open, is catalogued in [known-gaps.md](known-gaps.md). This file is the forward-looking half — what