From fa10c6dd74aac8ae95b2a92ec2eeb3142ec37903 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:45:00 -0500 Subject: [PATCH 1/7] perf(reader): the definitions pass builds only the fields it keeps of each Family and FamilySymbol record (#1335) --- .../changed/1335-definitions-projected.md | 8 + .../examples/probe_1335_projected_graphs.rs | 276 ++++++++++++++++ src/research/native_document.rs | 77 ++++- src/research/native_parameters.rs | 311 ++++++++++++++---- 4 files changed, 584 insertions(+), 88 deletions(-) create mode 100644 changelog.d/changed/1335-definitions-projected.md create mode 100644 probes/examples/probe_1335_projected_graphs.rs diff --git a/changelog.d/changed/1335-definitions-projected.md b/changelog.d/changed/1335-definitions-projected.md new file mode 100644 index 000000000..93fa55a6e --- /dev/null +++ b/changelog.d/changed/1335-definitions-projected.md @@ -0,0 +1,8 @@ +- **The parameter definitions pass no longer builds the values of the + `Family` and `FamilySymbol` records it reads two fields of (#1335).** The + pass decoded each of those records in full and kept the record's `m_id` and + its `m_categoryId` or `m_familyId`, which was most of the pass on a Revit 2015 + to 2023 project. It now reads them as before, with the same checks and the + same refusals, and builds only the fields it keeps + (`native_parameters::decode_graph_projected`, `Options::graph_fields`). No + export changes. The figures are in the pull request. diff --git a/probes/examples/probe_1335_projected_graphs.rs b/probes/examples/probe_1335_projected_graphs.rs new file mode 100644 index 000000000..34775d195 --- /dev/null +++ b/probes/examples/probe_1335_projected_graphs.rs @@ -0,0 +1,276 @@ +//! #1335: a projected graph decode is the full decode with fewer values built. +//! +//! FACT under test: `native_parameters::decode_graph_projected(.., keep)` +//! succeeds exactly where `decode_graph_with_catalog_usage` does and fails +//! with the same error where that fails; where it succeeds the graph is the +//! full one's in everything but the values of the fields not in `keep`: the +//! same objects (class, token, start, end of fields), edges, consumed bytes and +//! usage, and each object's fields exactly the full object's entries whose key +//! is in `keep`. The definitions pass reads a `Family` record's `m_id` and +//! `m_categoryId` and a `FamilySymbol` record's `m_id` and `m_familyId` and +//! nothing else, so its binding context (`symbol_families`, +//! `family_categories`) and its refusals are the same read from either graph. +//! +//! Every record of the file's native partitions is decoded in full and +//! projected under each of `KEEPS`, whatever its class: the lists are the ones +//! the definitions pass, the wall type and top join read and the phase read use, +//! the identity alone, and the empty list (every field left out). +//! +//! Output, one line each: +//! file release +//! compared records decodes full-ok full-refused mismatches +//! budget-stopped +//! keep compared mismatches +//! refused (a full decode that failed, by class) +//! binding-context family-categories symbol-families same +//! ingest-refusals same-refusals +//! mismatch (the first few) +//! +//! Usage: +//! cargo run --profile ci -p rvt-probes --example probe_1335_projected_graphs -- FILE +use anyhow::Result; +use rvt::RevitFile; +use rvt::research::native_document::{self, Options}; +use rvt::research::native_es_catalog; +use rvt::research::native_parameter_definitions::Registry as Definitions; +use rvt::research::native_parameters::{ + GraphLimits, ObjectGraph, decode_graph_projected, decode_graph_with_catalog_usage, +}; +use rvt::research::schema_registry; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::{Duration, Instant}; + +const BUDGET: Duration = Duration::from_secs(12 * 60); +const SHOWN: usize = 12; + +const KEEPS: [&[&str]; 6] = [ + &["m_id", "m_categoryId"], + &["m_id", "m_familyId"], + &["m_id", "m_WallAttributesId", "m_function", "m_baseOrTop"], + &[ + "m_id", + "m_createdPhaseId", + "m_demolishedPhaseId", + "m_phaseId", + ], + &["m_id"], + &[], +]; + +fn filtered(fields: &Value, keep: &[&str]) -> Value { + match fields { + Value::Object(map) => Value::Object( + map.iter() + .filter(|(key, _)| keep.contains(&key.as_str())) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + ), + other => other.clone(), + } +} + +fn identifier(value: &Value) -> Option { + if let Some(n) = value.as_i64() { + return Some(n); + } + ["m_id", "m_id64"] + .iter() + .find_map(|key| value.get(*key)) + .and_then(identifier) +} + +/// What the definitions pass does with a Family or FamilySymbol graph: the +/// identity check of `extract_records`, then `ingest_binding_context`. +fn ingest(definitions: &mut Definitions, graph: &ObjectGraph, id: u64) -> Result<(), String> { + let root = graph.objects.first().ok_or("empty owner graph")?; + let saved = identifier(&root.fields["m_id"]).ok_or("unsupported serialized ElementId value")?; + if u64::try_from(saved).ok() != Some(id) { + return Err(format!( + "serialized owner identity disagrees with indexed record {id}" + )); + } + definitions + .ingest_binding_context(graph) + .map_err(|error| error.to_string()) +} + +fn differences(full: &ObjectGraph, projected: &ObjectGraph, keep: &[&str]) -> Vec<&'static str> { + let mut found = Vec::new(); + if full.consumed_bytes != projected.consumed_bytes { + found.push("consumed"); + } + if full.objects.len() != projected.objects.len() { + found.push("object-count"); + return found; + } + for (a, b) in full.objects.iter().zip(&projected.objects) { + if a.class_tag != b.class_tag + || a.class_name != b.class_name + || a.token != b.token + || a.start != b.start + || a.fields_end != b.fields_end + { + found.push("object"); + } + let expected = if a.class_name == "ExtensibleStorageEntity" { + a.fields.clone() + } else { + filtered(&a.fields, keep) + }; + if expected != b.fields { + found.push("fields"); + } + } + let same_edges = full.edges.len() == projected.edges.len() + && full.edges.iter().zip(&projected.edges).all(|(a, b)| { + a.source_object_index == b.source_object_index + && a.pointer_offset == b.pointer_offset + && a.pointer_token == b.pointer_token + && a.target_object_index == b.target_object_index + && a.target_class_tag == b.target_class_tag + }); + if !same_edges { + found.push("edges"); + } + found +} + +fn main() -> Result<()> { + let path = std::env::args() + .nth(1) + .ok_or_else(|| anyhow::anyhow!("usage: probe_1335_projected_graphs FILE"))?; + let mut rf = RevitFile::open(&path)?; + let release = rf.layout_release()?; + println!("file {path} release {release}"); + if !native_document::NATIVE_RELEASES.contains(&release) { + println!("native: release {release} is outside the native readers' range"); + return Ok(()); + } + let formats = native_document::read_single(&mut rf, "Formats/Latest")?; + let registry = schema_registry::parse(&formats)?; + let catalog = native_document::read_single(&mut rf, "Global/Latest") + .and_then(|bytes| native_es_catalog::decode(&bytes, ®istry)) + .ok() + .map(|(catalog, _)| catalog); + let limits = GraphLimits { + max_values: 100_000, + max_objects: 100_000, + ..GraphLimits::default() + }; + // Every record is handed over with its body and no graph: this probe + // decodes them itself. + let options = Options { + releases: native_document::NATIVE_RELEASES, + graph_classes: Some(BTreeSet::new()), + keep_body: true, + ..Options::default() + }; + let started = Instant::now(); + let mut records = 0usize; + let mut decodes = 0usize; + let mut full_ok = 0usize; + let mut full_refused = 0usize; + let mut mismatches = 0usize; + let mut stopped = false; + let mut per_keep = [(0usize, 0usize); KEEPS.len()]; + let mut refused: BTreeMap = BTreeMap::new(); + let mut full_definitions = Definitions::default(); + let mut projected_definitions = Definitions::default(); + let mut ingest_refusals = 0usize; + let mut same_refusals = true; + let mut shown = 0usize; + let result = native_document::extract_graphs(&mut rf, &options, |record| { + if started.elapsed() > BUDGET { + stopped = true; + anyhow::bail!("time budget after {records} records"); + } + let Some(body) = &record.body else { + return Ok(()); + }; + records += 1; + let class = record.class_name.clone().unwrap_or_default(); + let full = decode_graph_with_catalog_usage(body, ®istry, &limits, catalog.as_ref()); + match &full { + Ok(_) => full_ok += 1, + Err(_) => { + full_refused += 1; + *refused.entry(class.clone()).or_default() += 1; + } + } + for (index, keep) in KEEPS.iter().enumerate() { + let projected = + decode_graph_projected(body, ®istry, &limits, catalog.as_ref(), keep); + decodes += 1; + per_keep[index].0 += 1; + let kind = match (&full, &projected) { + (Ok((graph, usage)), Ok((other, other_usage))) => { + if usage.values != other_usage.values || usage.objects != other_usage.objects { + Some("usage".to_string()) + } else { + differences(graph, other, keep) + .first() + .map(|k| k.to_string()) + } + } + (Err(a), Err(b)) => (a.to_string() != b.to_string()).then(|| "error".to_string()), + (Ok(_), Err(_)) => Some("projected-refused".to_string()), + (Err(_), Ok(_)) => Some("projected-accepted".to_string()), + }; + if let Some(kind) = kind { + mismatches += 1; + per_keep[index].1 += 1; + if shown < SHOWN { + shown += 1; + println!("mismatch {kind} {class}"); + } + } + } + if matches!(class.as_str(), "Family" | "FamilySymbol") { + let keep: &[&str] = if class == "Family" { + KEEPS[0] + } else { + KEEPS[1] + }; + let projected = + decode_graph_projected(body, ®istry, &limits, catalog.as_ref(), keep); + let id = record.identity.element_id; + let a = match &full { + Ok((graph, _)) => ingest(&mut full_definitions, graph, id), + Err(error) => Err(error.to_string()), + }; + let b = match &projected { + Ok((graph, _)) => ingest(&mut projected_definitions, graph, id), + Err(error) => Err(error.to_string()), + }; + ingest_refusals += usize::from(a.is_err()); + same_refusals &= a == b; + } + Ok(()) + }); + if let Err(error) = &result { + if !stopped { + println!("native: {error:#}"); + } + } + println!( + "compared records {records} decodes {decodes} full-ok {full_ok} full-refused {full_refused} mismatches {mismatches} budget-stopped {stopped}" + ); + for (keep, (compared, wrong)) in KEEPS.iter().zip(per_keep) { + println!( + "keep {} compared {compared} mismatches {wrong}", + keep.join(",") + ); + } + for (class, n) in &refused { + println!("refused {class} {n}"); + } + let same = full_definitions.family_categories == projected_definitions.family_categories + && full_definitions.symbol_families == projected_definitions.symbol_families; + println!( + "binding-context family-categories {} symbol-families {} same {same} ingest-refusals {ingest_refusals} same-refusals {same_refusals}", + full_definitions.family_categories.len(), + full_definitions.symbol_families.len(), + ); + Ok(()) +} diff --git a/src/research/native_document.rs b/src/research/native_document.rs index 5276e9e7e..a70afda83 100644 --- a/src/research/native_document.rs +++ b/src/research/native_document.rs @@ -95,6 +95,19 @@ pub struct Options { /// RE-342, which reads saved graphics on a file of Revit 2015 to 2022 with /// [`Self::releases`] widened, turns it on. pub deferred_history_episodes: bool, + /// For a record whose root class this names, the fields of its graph's + /// objects that are kept: those whose key (a bare field name, or + /// `Declaring::name` where the class chain declares it more than once) is + /// in the list, and no other ([`native_parameters::decode_graph_projected`]). + /// The record is read and checked as a full one is, so it is refused for + /// the same reasons, but the values of the fields left out are never built. + /// The list names `m_id` as well, which the check of a record against its + /// identity reads. A projected record has no [`Record::saved_metadata`], + /// which is made from the whole graph. A class the function gives `None` + /// for, and any record with no function here, keeps every field. For a + /// reader that takes only those fields from the records of those classes + /// (#1335). + pub graph_fields: Option Option<&'static [&'static str]>>, } /// The default [`Options::max_group_bytes`]: one reassembled continuation /// group's bytes. @@ -146,6 +159,7 @@ impl Default for Options { keep_body: false, value_source: None, deferred_history_episodes: false, + graph_fields: None, } } } @@ -574,6 +588,19 @@ pub(crate) type DefinitionKey = ( bool, ); +/// The fields of a `Family` and a `FamilySymbol` record that +/// [`crate::native_parameter_definitions::Registry::ingest_binding_context`] +/// reads, with `m_id`, which the record's identity is checked against: the +/// definitions pass takes nothing else from them (#1335). The rest of those +/// records is most of the pass. +fn binding_context_fields(class: &str) -> Option<&'static [&'static str]> { + match class { + "Family" => Some(&["m_id", "m_categoryId"]), + "FamilySymbol" => Some(&["m_id", "m_familyId"]), + _ => None, + } +} + /// The handle's [`DefinitionPass`] for `options`, read if it has none. fn definition_pass(file: &mut RevitFile, options: &Options) -> Result> { // Every field is named, so a new one has to be put in the key or left out @@ -594,6 +621,7 @@ fn definition_pass(file: &mut RevitFile, options: &Options) -> Result Result native_parameters::decode_graph_projected( + body, + registry, + &limits, + summary.extensible_storage_catalog.as_ref(), + keep, + ), + None => native_parameters::decode_graph_with_catalog_usage( + body, + registry, + &limits, + summary.extensible_storage_catalog.as_ref(), + ), + }; let decoded = decoded.and_then(|(graph, usage)| { if source.channel == 102 { let root = graph @@ -1184,13 +1226,14 @@ fn extract_records( .entry(diagnostic.clone().unwrap_or_default()) .or_default() += 1; } - let (saved_metadata, metadata_diagnostic) = match graph.as_ref().map(|graph| { - crate::native_metadata::project_with_definitions(graph, definitions) - }) { - Some(Ok(value)) => (Some(value), None), - Some(Err(error)) => (None, Some(error.to_string())), - None => (None, None), - }; + let (saved_metadata, metadata_diagnostic) = + match graph.as_ref().filter(|_| projected.is_none()).map(|graph| { + crate::native_metadata::project_with_definitions(graph, definitions) + }) { + Some(Ok(value)) => (Some(value), None), + Some(Err(error)) => (None, Some(error.to_string())), + None => (None, None), + }; summary.projected_metadata_records += usize::from(saved_metadata.is_some()); summary.unsupported_metadata_records += usize::from(metadata_diagnostic.is_some()); diff --git a/src/research/native_parameters.rs b/src/research/native_parameters.rs index 08ada846e..084889248 100644 --- a/src/research/native_parameters.rs +++ b/src/research/native_parameters.rs @@ -155,6 +155,35 @@ pub fn decode_graph_with_catalog_usage( registry: &Registry, limits: &GraphLimits, catalog: Option<&crate::native_extensible_storage::Catalog>, +) -> Result<(ObjectGraph, GraphUsage)> { + decode_graph_in(body, registry, limits, catalog, None) +} +/// [`decode_graph_with_catalog_usage`] keeping only the fields of every object +/// whose key is in `keep` (a bare field name, or `Declaring::name` where the +/// class chain declares the name more than once). Any other field is read and +/// checked as the whole decode reads and checks it, and left out of the +/// object's `fields`, which then hold exactly the entries of `keep` that the +/// full decode's do, with the same values. The graph is otherwise the full +/// decode's: it succeeds or fails with the same error, reads the same pointers +/// and objects, and has the same object starts and ends, edges, consumed bytes +/// and usage. A caller that reads only the fields in `keep` of a graph gets +/// what it gets from the full decode, without the values it ignores being +/// built and freed (#1335). +pub fn decode_graph_projected( + body: &[u8], + registry: &Registry, + limits: &GraphLimits, + catalog: Option<&crate::native_extensible_storage::Catalog>, + keep: &[&str], +) -> Result<(ObjectGraph, GraphUsage)> { + decode_graph_in(body, registry, limits, catalog, Some(keep)) +} +fn decode_graph_in( + body: &[u8], + registry: &Registry, + limits: &GraphLimits, + catalog: Option<&crate::native_extensible_storage::Catalog>, + keep: Option<&[&str]>, ) -> Result<(ObjectGraph, GraphUsage)> { ensure!( limits.max_values > 0 && limits.max_objects > 0 && limits.max_depth > 0, @@ -162,7 +191,7 @@ pub fn decode_graph_with_catalog_usage( ); let tag = le_bytes::u16_at(body, 0).ok_or_else(|| anyhow::anyhow!("truncated root object tag"))?; - let mut cursor = Cursor::new(body, registry, catalog, 2, *limits, Spans::None); + let mut cursor = Cursor::new(body, registry, catalog, 2, *limits, Spans::None).with_keep(keep); let mut objects = Vec::new(); let mut edges = Vec::new(); let mut shared = BTreeMap::::new(); @@ -186,7 +215,7 @@ pub fn decode_graph_with_catalog_usage( .name .clone(); let fields = cursor - .class(tag, 0) + .class_in::(tag, 0, true) .map_err(|e| anyhow::anyhow!("graph {name} starts{start}: {e}"))?; (name, fields) }; @@ -310,7 +339,7 @@ pub fn decode_object_fields_spanning( .name .clone(); let mut cursor = Cursor::new(body, registry, None, start, *limits, spans); - let fields = cursor.class(class_tag, 0)?; + let fields = cursor.class::(class_tag, 0)?; Ok(ObjectFields { class_tag, class_name, @@ -334,6 +363,15 @@ struct PendingPointer { struct ClassPlan<'a> { lineage: Vec<&'a Class>, keys: Vec>, + /// Whether each field is one a projected decode keeps + /// ([`decode_graph_projected`]); empty for a decode that is not one. + kept: Vec>, + /// The position, as (index in `lineage`, index in the class's fields), of + /// the first field whose key an earlier one in decoding order has already + /// taken: where a decode that builds no object stops with the error a + /// built one meets there. Found only for a projected decode; `None` + /// otherwise. + duplicate: Option<(usize, usize)>, } struct Cursor<'a> { body: &'a [u8], @@ -346,6 +384,8 @@ struct Cursor<'a> { keep_spans: Spans, plans: HashMap>>, limits: GraphLimits, + /// The fields a projected decode keeps ([`decode_graph_projected`]). + keep: Option<&'a [&'a str]>, } impl<'a> Cursor<'a> { fn new( @@ -367,8 +407,13 @@ impl<'a> Cursor<'a> { keep_spans, plans: HashMap::new(), limits, + keep: None, } } + fn with_keep(mut self, keep: Option<&'a [&'a str]>) -> Self { + self.keep = keep; + self + } fn take(&mut self, n: usize) -> Result<&[u8]> { let end = self .pos @@ -384,7 +429,7 @@ impl<'a> Cursor<'a> { fn u32(&mut self) -> Result { Ok(u32::from_le_bytes(self.take(4)?.try_into()?)) } - fn pointer(&mut self, external: bool) -> Result { + fn pointer(&mut self, external: bool) -> Result { let offset = self.pos; let token = self.u32()?; let tag = if token != 0 && !external { @@ -403,9 +448,13 @@ impl<'a> Cursor<'a> { } else { None }; - Ok(json!({"pointer_token":token,"class_tag":tag,"offset":offset})) + Ok(if BUILD { + json!({"pointer_token":token,"class_tag":tag,"offset":offset}) + } else { + Value::Null + }) } - fn entity_pointer(&mut self) -> Result { + fn entity_pointer(&mut self) -> Result { let offset = self.pos; let discriminator = i32::from_le_bytes(self.take(4)?.try_into()?); ensure!( @@ -413,7 +462,11 @@ impl<'a> Cursor<'a> { "unqualified ES entity discriminator {discriminator}" ); if discriminator == 0 { - return Ok(json!({"discriminator":0,"state":"serialized_absent","offset":offset})); + return Ok(if BUILD { + json!({"discriminator":0,"state":"serialized_absent","offset":offset}) + } else { + Value::Null + }); } let raw: [u8; 16] = self.take(16)?.try_into()?; let guid = crate::native_extensible_storage::guid_string(raw); @@ -427,9 +480,11 @@ impl<'a> Cursor<'a> { offset, schema_guid: Some(guid.clone()), }); - Ok( - json!({"discriminator":-1,"schema_guid":guid,"state":"serialized_present","offset":offset}), - ) + Ok(if BUILD { + json!({"discriminator":-1,"schema_guid":guid,"state":"serialized_present","offset":offset}) + } else { + Value::Null + }) } fn entity_fields(&mut self, guid: &str) -> Result { let schema = self @@ -503,7 +558,7 @@ impl<'a> Cursor<'a> { } modifier = 0x50; } - self.field( + self.field::( &Field { offset: 0, name: name.into(), @@ -520,12 +575,22 @@ impl<'a> Cursor<'a> { 0, ) } - fn class(&mut self, tag: u16, depth: usize) -> Result { + /// The object of class `tag` at the cursor. With `BUILD` unset nothing is + /// built: every byte is read, every check made and every pointer noted as + /// when it is set, and the value is `Null` (#1335). + fn class(&mut self, tag: u16, depth: usize) -> Result { + self.class_in::(tag, depth, false) + } + /// [`Self::class`], where `top` says the object is one the graph decode + /// itself starts on, not an inline one inside a field: only then are the + /// fields chosen by [`Self::keep`] the only ones built. + fn class_in(&mut self, tag: u16, depth: usize, top: bool) -> Result { ensure!( depth < self.limits.max_depth, "native class recursion budget exceeded" ); let plan = self.plan(tag, depth)?; + let project = BUILD && top && self.keep.is_some(); let mut values = Map::new(); for (distance, c) in plan.lineage.iter().enumerate().rev() { for (field_index, f) in c.fields.iter().enumerate() { @@ -546,33 +611,11 @@ impl<'a> Cursor<'a> { f.uninterpreted_flags ); let start = self.pos; - let value = if c.name == "ClassDefinitionRef" - && f.name == "m_ref" - && f.base == 10 - && f.modifier == 0 - { - // ClassDefinitionRef stores a 16-bit schema class tag. The - // qualified external reader performs Int16Reader -> ClassesContainer.find. - let class_tag = u16::from_le_bytes(self.take(2)?.try_into()?); - json!({"class_tag":class_tag,"class_name":self.registry.class(class_tag).map(|class|class.name.as_str())}) - } else if c.name == "ESEntity" - && f.name == "m_blob" - && f.base == 14 - && f.modifier == 1 - { - self.entity_pointer()? - } else if c.name == "VarSketchObj" - && f.name == "m_params" - && f.base == 14 - && f.modifier == 0x54 - { - // Qualified typeless owning-pointer vector: each entry still - // carries its concrete VarParam class tag in the wire data. - self.field_as(f, 0x51, f.array_count, depth + distance + 1)? + let kept = BUILD && (!project || plan.kept[distance][field_index]); + let value = if kept { + self.class_field::(c, f, depth, distance)? } else { - self.field(f, depth + distance + 1).map_err(|e| { - anyhow::anyhow!("{}.{} at {}: {e}", c.name, f.name, self.pos) - })? + self.class_field::(c, f, depth, distance)? }; if self.keeps_spans_of(c.tag) { self.spans.push(FieldSpan { @@ -582,17 +625,64 @@ impl<'a> Cursor<'a> { end: self.pos, }); } - let key = plan.keys[distance][field_index].clone(); - ensure!( - !values.contains_key(&key), - "ambiguous native field key {key} in declaring class {} (tag {})", - c.name, - c.tag - ); - values.insert(key, value); + if kept { + let key = plan.keys[distance][field_index].clone(); + ensure!( + !values.contains_key(&key), + "ambiguous native field key {key} in declaring class {} (tag {})", + c.name, + c.tag + ); + values.insert(key, value); + } else { + ensure!( + plan.duplicate != Some((distance, field_index)), + "ambiguous native field key {} in declaring class {} (tag {})", + plan.keys[distance][field_index], + c.name, + c.tag + ); + } } } - Ok(Value::Object(values)) + Ok(if BUILD { + Value::Object(values) + } else { + Value::Null + }) + } + /// One field `f` of class `c`, the `distance`th of the chain from the + /// object's own class, as [`Self::class_in`] reads it. + fn class_field( + &mut self, + c: &Class, + f: &Field, + depth: usize, + distance: usize, + ) -> Result { + if c.name == "ClassDefinitionRef" && f.name == "m_ref" && f.base == 10 && f.modifier == 0 { + // ClassDefinitionRef stores a 16-bit schema class tag. The + // qualified external reader performs Int16Reader -> ClassesContainer.find. + let class_tag = u16::from_le_bytes(self.take(2)?.try_into()?); + Ok(if BUILD { + json!({"class_tag":class_tag,"class_name":self.registry.class(class_tag).map(|class|class.name.as_str())}) + } else { + Value::Null + }) + } else if c.name == "ESEntity" && f.name == "m_blob" && f.base == 14 && f.modifier == 1 { + self.entity_pointer::() + } else if c.name == "VarSketchObj" + && f.name == "m_params" + && f.base == 14 + && f.modifier == 0x54 + { + // Qualified typeless owning-pointer vector: each entry still + // carries its concrete VarParam class tag in the wire data. + self.field_as::(f, 0x51, f.array_count, depth + distance + 1) + } else { + self.field::(f, depth + distance + 1) + .map_err(|e| anyhow::anyhow!("{}.{} at {}: {e}", c.name, f.name, self.pos)) + } } /// Whether the spans of the fields class `tag` declares are kept. fn keeps_spans_of(&self, tag: u16) -> bool { @@ -662,18 +752,59 @@ impl<'a> Cursor<'a> { .collect::>() }) .collect(); - let plan = Rc::new(ClassPlan { lineage, keys }); + let kept = match self.keep { + Some(keep) => lineage + .iter() + .zip(&keys) + .map(|(c, keys)| { + c.fields + .iter() + .zip(keys) + .map(|(f, key)| { + f.uninterpreted_flags & 2 == 0 && keep.contains(&key.as_str()) + }) + .collect::>() + }) + .collect(), + None => Vec::new(), + }; + // Only a projected decode reads fields without building them, and so + // needs the position; every other decode finds a repeated key in the + // object it builds. + let duplicate = if self.keep.is_some() { + let mut taken = std::collections::HashSet::new(); + let mut duplicate = None; + 'chain: for (distance, c) in lineage.iter().enumerate().rev() { + for (field_index, f) in c.fields.iter().enumerate() { + if f.uninterpreted_flags & 2 == 0 + && !taken.insert(keys[distance][field_index].as_str()) + { + duplicate = Some((distance, field_index)); + break 'chain; + } + } + } + duplicate + } else { + None + }; + let plan = Rc::new(ClassPlan { + lineage, + keys, + kept, + duplicate, + }); self.plans.insert(tag, Rc::clone(&plan)); Ok(plan) } - fn field(&mut self, f: &Field, depth: usize) -> Result { - self.field_as(f, f.modifier, f.array_count, depth) + fn field(&mut self, f: &Field, depth: usize) -> Result { + self.field_as::(f, f.modifier, f.array_count, depth) } /// [`Self::field`] reading `f` with `modifier` and `array_count` in place /// of its own: a container's items are `f` itself, one level down, which /// copying the field for each container was a share of a graph's /// allocations (#1237). - fn field_as( + fn field_as( &mut self, f: &Field, modifier: u8, @@ -692,12 +823,21 @@ impl<'a> Cursor<'a> { ); self.items += 1; match modifier { - 1..=3 if f.base == 14 => self.pointer(modifier == 3), + 1..=3 if f.base == 14 => self.pointer::(modifier == 3), 0x60 if f.base == 8 => { let n = self.u32()? as usize; ensure!(n <= 1_000_000, "native string unit budget exceeded"); - let units = le_bytes::u16_units(self.take(n * 2)?).collect::>(); - Ok(Value::String(String::from_utf16(&units)?)) + let bytes = self.take(n * 2)?; + if BUILD { + let units = le_bytes::u16_units(bytes).collect::>(); + Ok(Value::String(String::from_utf16(&units)?)) + } else { + if char::decode_utf16(le_bytes::u16_units(bytes)).any(|unit| unit.is_err()) { + let units = le_bytes::u16_units(bytes).collect::>(); + String::from_utf16(&units)?; + } + Ok(Value::Null) + } } 0x10 | 0x11 | 0x12 | 0x13 | 0x50 | 0x51 | 0x52 | 0x53 => { let n = if modifier < 0x50 { @@ -706,35 +846,64 @@ impl<'a> Cursor<'a> { self.u32()? }; ensure!(n <= 100_000, "native container count budget exceeded"); - let mut values = Vec::with_capacity(n.min(1024) as usize); - for _ in 0..n { - values.push(self.field_as(f, modifier & 0xf, None, depth + 1)?); + if BUILD { + let mut values = Vec::with_capacity(n.min(1024) as usize); + for _ in 0..n { + values.push(self.field_as::(f, modifier & 0xf, None, depth + 1)?); + } + Ok(Value::Array(values)) + } else { + for _ in 0..n { + self.field_as::(f, modifier & 0xf, None, depth + 1)?; + } + Ok(Value::Null) } - Ok(Value::Array(values)) } 0 => match f.base { 1 => { let b = self.take(1)?[0]; ensure!(b <= 1, "invalid native bool"); - Ok(json!(b != 0)) + Ok(if BUILD { json!(b != 0) } else { Value::Null }) + } + 2 => { + let b = self.take(1)?[0]; + Ok(if BUILD { json!(b) } else { Value::Null }) + } + 3 => { + let n = i16::from_le_bytes(self.take(2)?.try_into()?); + Ok(if BUILD { json!(n) } else { Value::Null }) + } + 4 => { + let n = i32::from_le_bytes(self.take(4)?.try_into()?); + Ok(if BUILD { json!(n) } else { Value::Null }) + } + 5 => { + let n = self.u32()?; + Ok(if BUILD { json!(n) } else { Value::Null }) } - 2 => Ok(json!(self.take(1)?[0])), - 3 => Ok(json!(i16::from_le_bytes(self.take(2)?.try_into()?))), - 4 => Ok(json!(i32::from_le_bytes(self.take(4)?.try_into()?))), - 5 => Ok(json!(self.u32()?)), 6 => { let n = f32::from_le_bytes(self.take(4)?.try_into()?); ensure!(n.is_finite(), "nonfinite native float"); - Ok(json!(n)) + Ok(if BUILD { json!(n) } else { Value::Null }) } 7 => { let n = f64::from_le_bytes(self.take(8)?.try_into()?); ensure!(n.is_finite(), "nonfinite native double"); - Ok(json!(n)) + Ok(if BUILD { json!(n) } else { Value::Null }) + } + 9 => { + let bytes = self.take(16)?; + Ok(if BUILD { + json!({"guid_bytes":bytes}) + } else { + Value::Null + }) + } + 11 => { + let n = i64::from_le_bytes(self.take(8)?.try_into()?); + Ok(if BUILD { json!(n) } else { Value::Null }) } - 9 => Ok(json!({"guid_bytes":self.take(16)?})), - 11 => Ok(json!(i64::from_le_bytes(self.take(8)?.try_into()?))), - 13 => self.field( + 13 => self.field::( f.nested_descriptor .as_deref() .ok_or_else(|| anyhow::anyhow!("nested schema absent"))?, @@ -745,7 +914,7 @@ impl<'a> Cursor<'a> { f.references.len() == 1, "inline native reference schema count" ); - self.class(f.references[0].tag, depth + 1) + self.class::(f.references[0].tag, depth + 1) } _ => anyhow::bail!("unsupported native scalar {}", f.base), }, @@ -766,7 +935,7 @@ fn identifier(value: &Value) -> Result { pub fn decode(body: &[u8], registry: &Registry) -> Result { let base = crate::native_element::decode_base(body, registry)?; let mut cursor = Cursor::new(body, registry, None, 2, GraphLimits::default(), Spans::None); - let fields = cursor.class(base.class_tag, 0)?; + let fields = cursor.class::(base.class_tag, 0)?; let derived_fields_end = cursor.pos; let mut parameters = Vec::new(); for (name, class_name, storage) in [ @@ -797,7 +966,7 @@ pub fn decode(body: &[u8], registry: &Registry) -> Result { registry.class(tag).is_some_and(|c| c.name == class_name), "parameter set class mismatch" ); - let set = cursor.class(tag, 0)?; + let set = cursor.class::(tag, 0)?; let pairs = set["m_paramSet"] .as_array() .ok_or_else(|| anyhow::anyhow!("parameter pair container absent"))?; From f5bee766c948dc9f3d85e617c6ef4ce71d3790b9 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:48:32 -0500 Subject: [PATCH 2/7] perf(reader): name the type of Options::graph_fields (#1335) --- src/research/native_document.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/research/native_document.rs b/src/research/native_document.rs index a70afda83..1225731a6 100644 --- a/src/research/native_document.rs +++ b/src/research/native_document.rs @@ -25,6 +25,10 @@ pub struct ResourceBudget { pub max_graph_objects: usize, } +/// The fields to keep of a record's graph, by its root class +/// ([`Options::graph_fields`]). +pub type GraphFields = fn(&str) -> Option<&'static [&'static str]>; + #[derive(Debug, Clone)] pub struct Options { pub selected_ids: BTreeSet, @@ -107,7 +111,7 @@ pub struct Options { /// for, and any record with no function here, keeps every field. For a /// reader that takes only those fields from the records of those classes /// (#1335). - pub graph_fields: Option Option<&'static [&'static str]>>, + pub graph_fields: Option, } /// The default [`Options::max_group_bytes`]: one reassembled continuation /// group's bytes. From 16527cea4945e36a87bf5bb8f0d36f48b7ebbc7a Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:49:48 -0500 Subject: [PATCH 3/7] perf(reader): the wall records read builds only the fields it keeps of each SWall and ArcWall graph (#1335) --- changelog.d/changed/1335-walls-projected.md | 6 ++++++ probes/examples/probe_1335_projected_graphs.rs | 3 ++- src/research/native_walls.rs | 13 +++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changelog.d/changed/1335-walls-projected.md diff --git a/changelog.d/changed/1335-walls-projected.md b/changelog.d/changed/1335-walls-projected.md new file mode 100644 index 000000000..0fc540bac --- /dev/null +++ b/changelog.d/changed/1335-walls-projected.md @@ -0,0 +1,6 @@ +- **The read of the wall records no longer builds the values of the fields it + ignores (#1335).** The read that takes each wall's type and whether its top + is joined to a roof or floor decoded every `SWall` and `ArcWall` graph in + full to read two fields of it. It reads and checks the same bytes, with the + same refusals, and builds only the fields it keeps. No export changes. The + figures are in the pull request. diff --git a/probes/examples/probe_1335_projected_graphs.rs b/probes/examples/probe_1335_projected_graphs.rs index 34775d195..5c2b24c13 100644 --- a/probes/examples/probe_1335_projected_graphs.rs +++ b/probes/examples/probe_1335_projected_graphs.rs @@ -44,9 +44,10 @@ use std::time::{Duration, Instant}; const BUDGET: Duration = Duration::from_secs(12 * 60); const SHOWN: usize = 12; -const KEEPS: [&[&str]; 6] = [ +const KEEPS: [&[&str]; 7] = [ &["m_id", "m_categoryId"], &["m_id", "m_familyId"], + &["m_id", "m_WallAttributesId", "m_baseOrTop"], &["m_id", "m_WallAttributesId", "m_function", "m_baseOrTop"], &[ "m_id", diff --git a/src/research/native_walls.rs b/src/research/native_walls.rs index 865c20445..bc59b9cf0 100644 --- a/src/research/native_walls.rs +++ b/src/research/native_walls.rs @@ -365,6 +365,18 @@ fn is_top_joined(graph: &crate::native_parameters::ObjectGraph) -> bool { }) } +/// The fields of an `SWall` and an `ArcWall` record that [`WallReads::read`] +/// takes: the type the record names ([`wall_type_id`]), whether a +/// `JoinToRoofGStep` joins its top ([`is_top_joined`]), and `m_id`, which the +/// record's identity is checked against. The rest of a wall's graph, most of +/// the read, is read and checked but not built (#1335). +fn wall_read_fields(class: &str) -> Option<&'static [&'static str]> { + match class { + "SWall" | "ArcWall" => Some(&["m_id", "m_WallAttributesId", "m_baseOrTop"]), + _ => None, + } +} + /// What reads of walls' native records have found, kept so that the reader /// of the same walls that follows finds it rather than reading the records /// again (#1237: each read is a pass over the file, and on a Revit 2015 to @@ -407,6 +419,7 @@ impl WallReads { let options = Options { selected_ids: selected, releases: crate::native_document::NATIVE_RELEASES, + graph_fields: Some(wall_read_fields), ..Options::default() }; let mut type_ids: BTreeMap = BTreeMap::new(); From 8bf59affc178c8a82a76a9720d957dec90cbed86 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:53:23 -0500 Subject: [PATCH 4/7] perf(reader): the native preamble finds each index leaf once, formats hex without a string per byte and reads the catalog's document without building what it ignores (#1335) --- changelog.d/changed/1335-preamble.md | 8 +++ src/research/native_document.rs | 2 +- src/research/native_es_catalog.rs | 14 ++++- src/research/native_index.rs | 81 +++++++++++++++++++++------- src/research/schema_registry.rs | 13 ++--- 5 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 changelog.d/changed/1335-preamble.md diff --git a/changelog.d/changed/1335-preamble.md b/changelog.d/changed/1335-preamble.md new file mode 100644 index 000000000..ce7b0df42 --- /dev/null +++ b/changelog.d/changed/1335-preamble.md @@ -0,0 +1,8 @@ +- **The native readers' shared preamble is read in less time (#1335).** Before + its first pass over a file's records every native reader reads the schema, + the history, the element index and the extensible storage catalog. The + element index no longer searches its layout for the leaves of each of its + thousands of rows, the schema's checksums no longer format a string for each + byte, and the catalog reads the document it takes its schemas from without + building the values of the fields it ignores. No export changes. The figures + are in the pull request. diff --git a/src/research/native_document.rs b/src/research/native_document.rs index 1225731a6..31cfffb75 100644 --- a/src/research/native_document.rs +++ b/src/research/native_document.rs @@ -303,7 +303,7 @@ pub struct StreamEnvelope { } /// `bytes` as lowercase hex, two digits a byte: `format!("{byte:02x}")` for /// each, without a `String` for every one (#1237). -fn lower_hex(bytes: &[u8]) -> String { +pub(crate) fn lower_hex(bytes: &[u8]) -> String { const DIGITS: &[u8; 16] = b"0123456789abcdef"; let mut hex = String::with_capacity(bytes.len() * 2); for &byte in bytes { diff --git a/src/research/native_es_catalog.rs b/src/research/native_es_catalog.rs index a36c42ad6..e1ef3307d 100644 --- a/src/research/native_es_catalog.rs +++ b/src/research/native_es_catalog.rs @@ -20,13 +20,23 @@ pub fn decode(bytes: &[u8], registry: &Registry) -> Result<(Catalog, Value)> { bytes[end..] == [0; 4], "ES global fixed footer is not the qualified zero value" ); - let graph = native_parameters::decode_graph_with_limits( + // What is read below: the root's pointer to the app info manager, its + // array of pointers, and the schema storage's two maps. Every other field + // of the document is read and checked, not built (#1335). + let (graph, _) = native_parameters::decode_graph_projected( &bytes[..end], registry, &GraphLimits { max_values: 2_000_000, ..Default::default() }, + None, + &[ + "m_pAppInfoManager", + "m_appInfoArr", + "m_storedSchemas", + "m_schemaUsageMap", + ], )?; let root = graph .objects @@ -122,7 +132,7 @@ pub fn decode(bytes: &[u8], registry: &Registry) -> Result<(Catalog, Value)> { } Ok(( catalog, - json!({"stream":"Global/Latest","inflated_sha256":Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect::(),"graph_bytes":end,"stream_bytes":bytes.len(),"fixed_footer":0,"root_class":"ADocument","app_info_object_index":manager,"storage_object_index":storages[0],"storage_start":storage.start,"storage_end":storage.fields_end,"complete_global_graph":true,"scope":"Owned file-local schemas; general nonzero fixed footers remain unqualified"}), + json!({"stream":"Global/Latest","inflated_sha256":crate::native_document::lower_hex(&Sha256::digest(bytes)),"graph_bytes":end,"stream_bytes":bytes.len(),"fixed_footer":0,"root_class":"ADocument","app_info_object_index":manager,"storage_object_index":storages[0],"storage_start":storage.start,"storage_end":storage.fields_end,"complete_global_graph":true,"scope":"Owned file-local schemas; general nonzero fixed footers remain unqualified"}), )) } fn string(v: &Value, key: &str) -> Result { diff --git a/src/research/native_index.rs b/src/research/native_index.rs index ab63b037b..6e9f16a53 100644 --- a/src/research/native_index.rs +++ b/src/research/native_index.rs @@ -419,16 +419,34 @@ fn fixed_layout(registry: &Registry, tag: u16) -> Result<(usize, Vec)> { visit(registry, tag, "", &mut size, &mut leaves, 0)?; Ok((size, leaves)) } -fn leaf_value(row: &[u8], layout: &[Leaf], prefix: &str) -> Result<(u64, usize)> { - let candidates: Vec<_> = layout - .iter() - .filter(|f| f.path.starts_with(prefix)) - .collect(); - ensure!( - candidates.len() == 1, - "ambiguous/missing index field {prefix}" - ); - let f = candidates[0]; +/// Which leaf of the layout each lookup of a row names, once found. The layout +/// is the same for every row, so the leaf a prefix names is found at the first +/// row that asks for it, where a lookup that fails fails as it always has, and +/// is not searched for again at each of the file's thousands of rows (#1335). +type LeafSlots = [Option; 7]; +fn leaf_value( + row: &[u8], + layout: &[Leaf], + slots: &mut LeafSlots, + slot: usize, + prefix: &str, +) -> Result<(u64, usize)> { + let at = match slots[slot] { + Some(at) => at, + None => { + let mut matching = layout + .iter() + .enumerate() + .filter(|(_, f)| f.path.starts_with(prefix)); + let at = match (matching.next(), matching.next()) { + (Some((at, _)), None) => at, + _ => anyhow::bail!("ambiguous/missing index field {prefix}"), + }; + slots[slot] = Some(at); + at + } + }; + let f = &layout[at]; let bytes = row .get(f.offset..f.offset + f.size) .ok_or_else(|| anyhow::anyhow!("truncated index field"))?; @@ -470,22 +488,47 @@ pub fn parse(index: &[u8], registry: &Registry, episodes: &[String]) -> Result

Reader<'a> { "schema opaque trailer budget exceeded" ); let opaque_entries_offset = self.pos; - let opaque_entries_sha256 = - Sha256::digest(self.take(opaque_16byte_entry_count as usize * 16)?) - .iter() - .map(|b| format!("{b:02x}")) - .collect::(); + let opaque_entries_sha256 = crate::native_document::lower_hex(&Sha256::digest( + self.take(opaque_16byte_entry_count as usize * 16)?, + )); self.classes[ordinal] = Some(Class { tag, name, @@ -296,10 +294,7 @@ pub fn parse(bytes: &[u8]) -> Result { .collect::>>() .ok_or_else(|| anyhow::anyhow!("incomplete schema definition"))?; Ok(Registry { - source_sha256: Sha256::digest(bytes) - .iter() - .map(|b| format!("{b:02x}")) - .collect::(), + source_sha256: crate::native_document::lower_hex(&Sha256::digest(bytes)), consumed_bytes: r.pos, reference_count: r.references, terminator_offset, From 76e98d7bf0435ae5a1f321620f71be7e2723da80 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:54:06 -0500 Subject: [PATCH 5/7] probe(1335): the catalog's document is compared in full and projected --- .../examples/probe_1335_projected_graphs.rs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/probes/examples/probe_1335_projected_graphs.rs b/probes/examples/probe_1335_projected_graphs.rs index 34775d195..4311a5d83 100644 --- a/probes/examples/probe_1335_projected_graphs.rs +++ b/probes/examples/probe_1335_projected_graphs.rs @@ -18,6 +18,9 @@ //! //! Output, one line each: //! file release +//! global-graph +//! (the Global/Latest document the catalog is read from, in full and +//! under the fields the catalog reads) //! compared records decodes full-ok full-refused mismatches //! budget-stopped //! keep compared mismatches @@ -158,6 +161,42 @@ fn main() -> Result<()> { max_objects: 100_000, ..GraphLimits::default() }; + // The document the extensible storage catalog is read from, in full and + // under the fields the catalog reads: the same graph either way. + let global_keep = [ + "m_pAppInfoManager", + "m_appInfoArr", + "m_storedSchemas", + "m_schemaUsageMap", + ]; + match native_document::read_single(&mut rf, "Global/Latest") { + Ok(bytes) if bytes.len() >= 6 => { + let document = &bytes[..bytes.len() - 4]; + let wide = GraphLimits { + max_values: 2_000_000, + ..GraphLimits::default() + }; + let full = decode_graph_with_catalog_usage(document, ®istry, &wide, None); + let projected = decode_graph_projected(document, ®istry, &wide, None, &global_keep); + let verdict = match (&full, &projected) { + (Ok((a, usage)), Ok((b, other))) => { + if usage.values != other.values || usage.objects != other.objects { + "usage" + } else { + differences(a, b, &global_keep) + .first() + .copied() + .unwrap_or("same") + } + } + (Err(a), Err(b)) if a.to_string() == b.to_string() => "same-refusal", + (Err(_), Err(_)) => "error", + _ => "accepted-differently", + }; + println!("global-graph {verdict}"); + } + _ => println!("global-graph absent"), + } // Every record is handed over with its body and no graph: this probe // decodes them itself. let options = Options { From 4140b96c0dd3407ef32e4fb11284d6be005147d6 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:46:08 -0500 Subject: [PATCH 6/7] perf(reader): each projected field list is a constant beside its one reader, the probe imports them, and the index leaves are named fields (#1335) --- docs/maintaining.md | 14 ++ .../examples/probe_1335_projected_graphs.rs | 150 ++++++++++-------- src/research/native_document.rs | 25 ++- src/research/native_es_catalog.rs | 62 +++++--- src/research/native_index.rs | 58 ++++--- src/research/native_parameter_definitions.rs | 24 +++ src/research/native_wall_types.rs | 5 +- src/research/native_walls.rs | 52 ++++-- 8 files changed, 261 insertions(+), 129 deletions(-) diff --git a/docs/maintaining.md b/docs/maintaining.md index 955eee630..c3498381e 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -637,6 +637,20 @@ save it (a pull request's run does not: its caches are its own). A ref that is not a full SHA is built each time, the probe examples always are, and `clean_caches.py` keeps the newest sixteen. +## Projected graphs (#1335) + +A native pass that reads a few fields of a large graph can decode it projected +(`native_parameters::decode_graph_projected`, `native_document::Options::graph_fields`): +every byte is read and checked as in the full decode, so the record is refused for +the same reasons, but only the listed fields are built. Each list is a `pub const` +beside the one function that reads the projected graph +(`Registry::ingest_binding_context`, `native_walls::wall_facts`, +`native_es_catalog::from_graph`), and a field that function comes to read must be +added to its list, because a field left out reads as absent without an error. +`probes/examples/probe_1335_projected_graphs.rs` imports the constants, decodes every +record both ways and runs those functions on both graphs; run it +(`-f probe=probe_1335_projected_graphs`) after any change to a list or a reader. + ## Dependencies and the MSRV `rust-version` in `Cargo.toml` is the oldest stable Rust the crate builds on, and diff --git a/probes/examples/probe_1335_projected_graphs.rs b/probes/examples/probe_1335_projected_graphs.rs index 189417995..69f02ada1 100644 --- a/probes/examples/probe_1335_projected_graphs.rs +++ b/probes/examples/probe_1335_projected_graphs.rs @@ -6,27 +6,28 @@ //! full one's in everything but the values of the fields not in `keep`: the //! same objects (class, token, start, end of fields), edges, consumed bytes and //! usage, and each object's fields exactly the full object's entries whose key -//! is in `keep`. The definitions pass reads a `Family` record's `m_id` and -//! `m_categoryId` and a `FamilySymbol` record's `m_id` and `m_familyId` and -//! nothing else, so its binding context (`symbol_families`, -//! `family_categories`) and its refusals are the same read from either graph. +//! is in `keep`. And each reader that is given a projected graph reads the same +//! from it as from the full one: the definitions pass's +//! `Registry::ingest_binding_context` (the `Family` and `FamilySymbol` lists), +//! the wall read's `wall_facts` (`WALL_READ_FIELDS`) and the catalog's +//! `from_graph` (`CATALOG_FIELDS`). The lists are the readers' own constants, +//! imported, not copied. //! //! Every record of the file's native partitions is decoded in full and -//! projected under each of `KEEPS`, whatever its class: the lists are the ones -//! the definitions pass, the wall type and top join read and the phase read use, -//! the identity alone, and the empty list (every field left out). +//! projected under each of the readers' lists, whatever its class, and under +//! the identity alone and the empty list (every field left out). The graph the +//! catalog is read from, the `Global/Latest` document, is read both ways too. //! //! Output, one line each: //! file release -//! global-graph -//! (the Global/Latest document the catalog is read from, in full and -//! under the fields the catalog reads) +//! global-catalog //! compared records decodes full-ok full-refused mismatches //! budget-stopped //! keep compared mismatches //! refused (a full decode that failed, by class) //! binding-context family-categories symbol-families same //! ingest-refusals same-refusals +//! wall-facts compared same //! mismatch (the first few) //! //! Usage: @@ -34,11 +35,14 @@ use anyhow::Result; use rvt::RevitFile; use rvt::research::native_document::{self, Options}; -use rvt::research::native_es_catalog; -use rvt::research::native_parameter_definitions::Registry as Definitions; +use rvt::research::native_es_catalog::{self, CATALOG_FIELDS}; +use rvt::research::native_parameter_definitions::{ + FAMILY_BINDING_FIELDS, Registry as Definitions, SYMBOL_BINDING_FIELDS, +}; use rvt::research::native_parameters::{ GraphLimits, ObjectGraph, decode_graph_projected, decode_graph_with_catalog_usage, }; +use rvt::research::native_walls::{WALL_READ_FIELDS, wall_facts}; use rvt::research::schema_registry; use serde_json::Value; use std::collections::{BTreeMap, BTreeSet}; @@ -47,17 +51,13 @@ use std::time::{Duration, Instant}; const BUDGET: Duration = Duration::from_secs(12 * 60); const SHOWN: usize = 12; -const KEEPS: [&[&str]; 7] = [ - &["m_id", "m_categoryId"], - &["m_id", "m_familyId"], - &["m_id", "m_WallAttributesId", "m_baseOrTop"], - &["m_id", "m_WallAttributesId", "m_function", "m_baseOrTop"], - &[ - "m_id", - "m_createdPhaseId", - "m_demolishedPhaseId", - "m_phaseId", - ], +/// The readers' lists, then two that stress the mechanism: the identity alone +/// and no field at all. +const KEEPS: [&[&str]; 6] = [ + FAMILY_BINDING_FIELDS, + SYMBOL_BINDING_FIELDS, + WALL_READ_FIELDS, + CATALOG_FIELDS, &["m_id"], &[], ]; @@ -140,6 +140,48 @@ fn differences(full: &ObjectGraph, projected: &ObjectGraph, keep: &[&str]) -> Ve found } +/// The catalog `from_graph` reads from the `Global/Latest` document decoded in +/// full and decoded under `CATALOG_FIELDS`. +fn global_catalog(rf: &mut RevitFile, registry: &schema_registry::Registry) -> String { + let Ok(bytes) = native_document::read_single(rf, "Global/Latest") else { + return "absent".into(); + }; + let Ok(document) = native_es_catalog::document(&bytes) else { + return "absent".into(); + }; + let limits = native_es_catalog::document_limits(); + let full = decode_graph_with_catalog_usage(document, registry, &limits, None); + let projected = decode_graph_projected(document, registry, &limits, None, CATALOG_FIELDS); + match (&full, &projected) { + (Ok((a, usage)), Ok((b, other))) => { + if usage.values != other.values || usage.objects != other.objects { + return "usage".into(); + } + if let Some(kind) = differences(a, b, CATALOG_FIELDS).first() { + return format!("graph-{kind}"); + } + match ( + native_es_catalog::from_graph(a, &bytes), + native_es_catalog::from_graph(b, &bytes), + ) { + (Ok((c, v)), Ok((d, w))) => { + let same = serde_json::to_value(&c).ok() == serde_json::to_value(&d).ok(); + if same && v == w { + "same".into() + } else { + "catalog".into() + } + } + (Err(x), Err(y)) if x.to_string() == y.to_string() => "same-refusal".into(), + _ => "catalog-refusal".into(), + } + } + (Err(a), Err(b)) if a.to_string() == b.to_string() => "same-refusal".into(), + (Err(_), Err(_)) => "error".into(), + _ => "accepted-differently".into(), + } +} + fn main() -> Result<()> { let path = std::env::args() .nth(1) @@ -153,6 +195,7 @@ fn main() -> Result<()> { } let formats = native_document::read_single(&mut rf, "Formats/Latest")?; let registry = schema_registry::parse(&formats)?; + println!("global-catalog {}", global_catalog(&mut rf, ®istry)); let catalog = native_document::read_single(&mut rf, "Global/Latest") .and_then(|bytes| native_es_catalog::decode(&bytes, ®istry)) .ok() @@ -162,42 +205,6 @@ fn main() -> Result<()> { max_objects: 100_000, ..GraphLimits::default() }; - // The document the extensible storage catalog is read from, in full and - // under the fields the catalog reads: the same graph either way. - let global_keep = [ - "m_pAppInfoManager", - "m_appInfoArr", - "m_storedSchemas", - "m_schemaUsageMap", - ]; - match native_document::read_single(&mut rf, "Global/Latest") { - Ok(bytes) if bytes.len() >= 6 => { - let document = &bytes[..bytes.len() - 4]; - let wide = GraphLimits { - max_values: 2_000_000, - ..GraphLimits::default() - }; - let full = decode_graph_with_catalog_usage(document, ®istry, &wide, None); - let projected = decode_graph_projected(document, ®istry, &wide, None, &global_keep); - let verdict = match (&full, &projected) { - (Ok((a, usage)), Ok((b, other))) => { - if usage.values != other.values || usage.objects != other.objects { - "usage" - } else { - differences(a, b, &global_keep) - .first() - .copied() - .unwrap_or("same") - } - } - (Err(a), Err(b)) if a.to_string() == b.to_string() => "same-refusal", - (Err(_), Err(_)) => "error", - _ => "accepted-differently", - }; - println!("global-graph {verdict}"); - } - _ => println!("global-graph absent"), - } // Every record is handed over with its body and no graph: this probe // decodes them itself. let options = Options { @@ -219,6 +226,8 @@ fn main() -> Result<()> { let mut projected_definitions = Definitions::default(); let mut ingest_refusals = 0usize; let mut same_refusals = true; + let mut walls = 0usize; + let mut same_walls = true; let mut shown = 0usize; let result = native_document::extract_graphs(&mut rf, &options, |record| { if started.elapsed() > BUDGET { @@ -266,15 +275,16 @@ fn main() -> Result<()> { } } } + // The readers, on the graph they are given and on the full one. + let id = record.identity.element_id; if matches!(class.as_str(), "Family" | "FamilySymbol") { - let keep: &[&str] = if class == "Family" { - KEEPS[0] + let keep = if class == "Family" { + FAMILY_BINDING_FIELDS } else { - KEEPS[1] + SYMBOL_BINDING_FIELDS }; let projected = decode_graph_projected(body, ®istry, &limits, catalog.as_ref(), keep); - let id = record.identity.element_id; let a = match &full { Ok((graph, _)) => ingest(&mut full_definitions, graph, id), Err(error) => Err(error.to_string()), @@ -286,6 +296,19 @@ fn main() -> Result<()> { ingest_refusals += usize::from(a.is_err()); same_refusals &= a == b; } + if matches!(class.as_str(), "SWall" | "ArcWall") { + let projected = decode_graph_projected( + body, + ®istry, + &limits, + catalog.as_ref(), + WALL_READ_FIELDS, + ); + if let (Ok((graph, _)), Ok((other, _))) = (&full, &projected) { + walls += 1; + same_walls &= wall_facts(graph) == wall_facts(other); + } + } Ok(()) }); if let Err(error) = &result { @@ -312,5 +335,6 @@ fn main() -> Result<()> { full_definitions.family_categories.len(), full_definitions.symbol_families.len(), ); + println!("wall-facts compared {walls} same {same_walls}"); Ok(()) } diff --git a/src/research/native_document.rs b/src/research/native_document.rs index 31cfffb75..256fa5599 100644 --- a/src/research/native_document.rs +++ b/src/research/native_document.rs @@ -110,7 +110,11 @@ pub struct Options { /// which is made from the whole graph. A class the function gives `None` /// for, and any record with no function here, keeps every field. For a /// reader that takes only those fields from the records of those classes - /// (#1335). + /// (#1335): the list is a `pub const` beside the one function that reads + /// the projected graph (`wall_facts`, `Registry::ingest_binding_context`), + /// so that a field read is added to the list where it is read. The + /// definitions pass, which [`extract`] makes first, sets its own function + /// whatever this is. pub graph_fields: Option, } /// The default [`Options::max_group_bytes`]: one reassembled continuation @@ -592,19 +596,6 @@ pub(crate) type DefinitionKey = ( bool, ); -/// The fields of a `Family` and a `FamilySymbol` record that -/// [`crate::native_parameter_definitions::Registry::ingest_binding_context`] -/// reads, with `m_id`, which the record's identity is checked against: the -/// definitions pass takes nothing else from them (#1335). The rest of those -/// records is most of the pass. -fn binding_context_fields(class: &str) -> Option<&'static [&'static str]> { - match class { - "Family" => Some(&["m_id", "m_categoryId"]), - "FamilySymbol" => Some(&["m_id", "m_familyId"]), - _ => None, - } -} - /// The handle's [`DefinitionPass`] for `options`, read if it has none. fn definition_pass(file: &mut RevitFile, options: &Options) -> Result> { // Every field is named, so a new one has to be put in the key or left out @@ -649,7 +640,11 @@ fn definition_pass(file: &mut RevitFile, options: &Options) -> Result Result<(Catalog, Value)> { +/// The fields of the `Global/Latest` document's graph that [`from_graph`] +/// reads: the root's pointer to the app info manager, the manager's array of +/// pointers, and the schema storage's two maps. [`decode`] builds no other +/// field of the document (it reads and checks every one, #1335), and +/// [`from_graph`] is the only reader of that graph: a field it comes to read +/// must be added here. +pub const CATALOG_FIELDS: &[&str] = &[ + "m_pAppInfoManager", + "m_appInfoArr", + "m_storedSchemas", + "m_schemaUsageMap", +]; + +/// The resource budgets the document is decoded under. +pub fn document_limits() -> GraphLimits { + GraphLimits { + max_values: 2_000_000, + ..Default::default() + } +} + +/// The graph's bytes of a global stream, the stream without its fixed u32 +/// footer. Only the independently observed zero-footer envelope is qualified. +pub fn document(bytes: &[u8]) -> Result<&[u8]> { ensure!( bytes.len() >= 6 && bytes.len() <= 32 * 1024 * 1024, "ES global stream size unqualified" @@ -20,24 +41,27 @@ pub fn decode(bytes: &[u8], registry: &Registry) -> Result<(Catalog, Value)> { bytes[end..] == [0; 4], "ES global fixed footer is not the qualified zero value" ); - // What is read below: the root's pointer to the app info manager, its - // array of pointers, and the schema storage's two maps. Every other field - // of the document is read and checked, not built (#1335). + Ok(&bytes[..end]) +} + +/// The catalog of the global stream `bytes` and what is known of where it +/// was read from. Global object streams carry a fixed u32 footer after the +/// deferred graph. +pub fn decode(bytes: &[u8], registry: &Registry) -> Result<(Catalog, Value)> { let (graph, _) = native_parameters::decode_graph_projected( - &bytes[..end], + document(bytes)?, registry, - &GraphLimits { - max_values: 2_000_000, - ..Default::default() - }, + &document_limits(), None, - &[ - "m_pAppInfoManager", - "m_appInfoArr", - "m_storedSchemas", - "m_schemaUsageMap", - ], + CATALOG_FIELDS, )?; + from_graph(&graph, bytes) +} + +/// The catalog in the graph of `bytes`' document, built only of +/// [`CATALOG_FIELDS`] or whole: the same either way. +pub fn from_graph(graph: &ObjectGraph, bytes: &[u8]) -> Result<(Catalog, Value)> { + let end = bytes.len().saturating_sub(4); let root = graph .objects .first() @@ -46,7 +70,7 @@ pub fn decode(bytes: &[u8], registry: &Registry) -> Result<(Catalog, Value)> { root.class_name == "ADocument", "ES catalog requires ADocument root" ); - let manager = owned(&graph, 0, &root.fields["m_pAppInfoManager"])?; + let manager = owned(graph, 0, &root.fields["m_pAppInfoManager"])?; ensure!( graph.objects[manager].class_name == "AppInfoManager", "ES manager class mismatch" @@ -59,7 +83,7 @@ pub fn decode(bytes: &[u8], registry: &Registry) -> Result<(Catalog, Value)> { if pointer["pointer_token"].as_u64() == Some(0) { continue; } - let index = owned(&graph, manager, pointer)?; + let index = owned(graph, manager, pointer)?; if graph.objects[index].class_name == "ESSchemaStorage" { storages.push(index); } diff --git a/src/research/native_index.rs b/src/research/native_index.rs index 6e9f16a53..d4eeef1bc 100644 --- a/src/research/native_index.rs +++ b/src/research/native_index.rs @@ -419,19 +419,28 @@ fn fixed_layout(registry: &Registry, tag: u16) -> Result<(usize, Vec)> { visit(registry, tag, "", &mut size, &mut leaves, 0)?; Ok((size, leaves)) } -/// Which leaf of the layout each lookup of a row names, once found. The layout -/// is the same for every row, so the leaf a prefix names is found at the first -/// row that asks for it, where a lookup that fails fails as it always has, and -/// is not searched for again at each of the file's thousands of rows (#1335). -type LeafSlots = [Option; 7]; +/// Which leaf of the layout each lookup of a row names, once found: one field +/// for each lookup `parse` makes. The layout is the same for every row, so the +/// leaf a prefix names is found at the first row that asks for it, where a +/// lookup that fails fails as it always has, and is not searched for again at +/// each of the file's thousands of rows (#1335). +#[derive(Default)] +struct IndexLeaves { + id: Option, + original_id: Option, + creation: Option, + modification: Option, + user_modification: Option, + owner: Option, + partition: Option, +} fn leaf_value( row: &[u8], layout: &[Leaf], - slots: &mut LeafSlots, - slot: usize, + slot: &mut Option, prefix: &str, ) -> Result<(u64, usize)> { - let at = match slots[slot] { + let at = match *slot { Some(at) => at, None => { let mut matching = layout @@ -442,7 +451,7 @@ fn leaf_value( (Some((at, _)), None) => at, _ => anyhow::bail!("ambiguous/missing index field {prefix}"), }; - slots[slot] = Some(at); + *slot = Some(at); at } }; @@ -488,25 +497,30 @@ pub fn parse(index: &[u8], registry: &Registry, episodes: &[String]) -> Result

Result

Option<&'static [&'static str]> { + match class { + "Family" => Some(FAMILY_BINDING_FIELDS), + "FamilySymbol" => Some(SYMBOL_BINDING_FIELDS), + _ => None, + } +} + impl Registry { /// Retain only the native links needed to resolve category bindings. All IDs /// must come from the same current document namespace as the definitions. + /// + /// The definitions pass builds only [`FAMILY_BINDING_FIELDS`] of a `Family` + /// graph and [`SYMBOL_BINDING_FIELDS`] of a `FamilySymbol` graph (#1335), + /// and this is the only reader of either: a field read here must be added + /// to the list, or it reads as absent. pub fn ingest_binding_context(&mut self, graph: &ObjectGraph) -> Result<()> { let root = graph .objects diff --git a/src/research/native_wall_types.rs b/src/research/native_wall_types.rs index 193a92adc..e85e252df 100644 --- a/src/research/native_wall_types.rs +++ b/src/research/native_wall_types.rs @@ -16,7 +16,10 @@ use crate::{RevitFile, native_document}; use std::collections::{BTreeMap, BTreeSet}; /// The id of the type a wall's record names (`m_WallAttributesId`), in either -/// shape of `Identifier` (RE-283). +/// shape of `Identifier` (RE-283). Read through +/// [`crate::native_walls::wall_facts`] from a graph of which only +/// [`crate::native_walls::WALL_READ_FIELDS`] are built: the field it reads is +/// in that list, and one it comes to read must be added. pub(crate) fn wall_type_id(graph: &ObjectGraph) -> Option { graph.objects.iter().find_map(|object| { let id = identifier_i64(object.fields.get("m_WallAttributesId")?)?; diff --git a/src/research/native_walls.rs b/src/research/native_walls.rs index bc59b9cf0..958bee0c9 100644 --- a/src/research/native_walls.rs +++ b/src/research/native_walls.rs @@ -357,7 +357,9 @@ pub struct WallRecords { } /// Whether a wall's top is joined to a roof or floor: one of the -/// `JoinToRoofGStep` objects of its record has `m_baseOrTop` 1. +/// `JoinToRoofGStep` objects of its record has `m_baseOrTop` 1. Read from the +/// graph [`WallReads::read`] builds only [`WALL_READ_FIELDS`] of, through +/// [`wall_facts`]: a field read here is added to that list. fn is_top_joined(graph: &crate::native_parameters::ObjectGraph) -> bool { graph.objects.iter().any(|object| { object.class_name == "JoinToRoofGStep" @@ -365,14 +367,37 @@ fn is_top_joined(graph: &crate::native_parameters::ObjectGraph) -> bool { }) } -/// The fields of an `SWall` and an `ArcWall` record that [`WallReads::read`] -/// takes: the type the record names ([`wall_type_id`]), whether a -/// `JoinToRoofGStep` joins its top ([`is_top_joined`]), and `m_id`, which the -/// record's identity is checked against. The rest of a wall's graph, most of -/// the read, is read and checked but not built (#1335). +/// The fields of an `SWall` and an `ArcWall` graph that [`wall_facts`] reads +/// ([`wall_type_id`]: `m_WallAttributesId`; [`is_top_joined`]: `m_baseOrTop`), +/// and `m_id`, which the check of a record against its identity reads. +/// [`WallReads::read`] builds no other field of those graphs (#1335), so a field +/// either function comes to read must be added here. +pub const WALL_READ_FIELDS: &[&str] = &["m_id", "m_WallAttributesId", "m_baseOrTop"]; + +/// What [`WallReads::read`] takes from the graph of a wall's record. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WallFacts { + /// The type the record names ([`wall_type_id`]). + pub type_id: Option, + /// Whether a roof or floor is joined to the wall's top ([`is_top_joined`]). + pub top_joined: bool, +} + +/// [`WallFacts`] of a wall's graph: the only reader of the graph +/// [`WallReads::read`] projects to [`WALL_READ_FIELDS`] (the probe of #1335 +/// runs it on the full graph and on the projected one). +pub fn wall_facts(graph: &crate::native_parameters::ObjectGraph) -> WallFacts { + WallFacts { + type_id: wall_type_id(graph), + top_joined: is_top_joined(graph), + } +} + +/// The fields [`WallReads::read`] builds of a record of `class` +/// ([`crate::native_document::Options::graph_fields`]). fn wall_read_fields(class: &str) -> Option<&'static [&'static str]> { match class { - "SWall" | "ArcWall" => Some(&["m_id", "m_WallAttributesId", "m_baseOrTop"]), + "SWall" | "ArcWall" => Some(WALL_READ_FIELDS), _ => None, } } @@ -398,6 +423,14 @@ impl WallReads { /// of `type_walls` and whether the top of each of `join_walls` is joined /// ([`wall_top_joins`]), in one pass over the records of the walls not /// read already. Nothing is read where there are none. + /// + /// On Revit 2015 to 2023 this is the read of the wall records that + /// `read_wall_records`, `attach_wall_type_functions` and + /// `attach_wall_top_joins` share, and it builds only [`WALL_READ_FIELDS`] of + /// an `SWall` or `ArcWall` graph (#1335): a fact added to it that takes + /// another field must add the field to the list, not assume the whole + /// graph, and reads it through [`wall_facts`]. The wall cuts and + /// [`native_walls_and_top_joins`] read their own selections, in full. pub fn read( &mut self, rf: &mut RevitFile, @@ -429,15 +462,16 @@ impl WallReads { return Ok(()); }; let id = record.identity.element_id; + let facts = wall_facts(graph); if types.contains(&id) { - if let Some(type_id) = wall_type_id(graph) { + if let Some(type_id) = facts.type_id { type_ids.insert(id, type_id); } } if joins.contains(&id) && matches!(record.class_name.as_deref(), Some("SWall" | "ArcWall")) { - top_joined.insert(id, is_top_joined(graph)); + top_joined.insert(id, facts.top_joined); } Ok(()) })?; From 98566dd37f870297dc97531b34d12ac60579c75e Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:28:06 -0500 Subject: [PATCH 7/7] perf(reader): the named-parameter read searches a record for each id it looks for, not every four-byte window (#1335) --- changelog.d/changed/1335-needle-scan.md | 7 ++ src/research/native_document.rs | 93 ++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 changelog.d/changed/1335-needle-scan.md diff --git a/changelog.d/changed/1335-needle-scan.md b/changelog.d/changed/1335-needle-scan.md new file mode 100644 index 000000000..1b296b89f --- /dev/null +++ b/changelog.d/changed/1335-needle-scan.md @@ -0,0 +1,7 @@ +- **The read of the parameters named for a property searches each record for + the few definition ids it looks for, instead of scanning every four-byte + window (#1335).** On a Revit 2015 to 2023 project that read looks at about + 5,600 records for six ids. It now searches each record's bytes for each id + with the vectorised substring search, and keeps the window scan for more + than 16 ids. The records it decodes are the same. No export changes. The + figures are in the pull request. diff --git a/src/research/native_document.rs b/src/research/native_document.rs index 256fa5599..6b8c0911f 100644 --- a/src/research/native_document.rs +++ b/src/research/native_document.rs @@ -180,8 +180,17 @@ struct IdNeedles { /// before the set is asked. low: Vec, ids: BTreeSet, + /// A searcher for each id's four bytes while there are at most + /// [`MAX_FINDERS`] of them, so that a body is searched for each by the + /// vectorised substring search, not scanned a window at a time; empty for + /// more ids, and for none (#1335). + finders: Vec>, } +/// The most ids [`IdNeedles`] searches for one by one: past it, a body is +/// scanned once, a window at a time. +const MAX_FINDERS: usize = 16; + impl IdNeedles { /// The word and bit of `low` that `value`'s low 16 bits select. fn slot(value: u32) -> (usize, u64) { @@ -196,11 +205,28 @@ impl IdNeedles { let (word, bit) = Self::slot(id); low[word] |= bit; } - Self { low, ids } + let finders = if ids.len() <= MAX_FINDERS { + ids.iter() + .map(|id| memchr::memmem::Finder::new(&id.to_le_bytes()).into_owned()) + .collect() + } else { + Vec::new() + }; + Self { low, ids, finders } } /// Whether any of the ids' four bytes is anywhere in `body`. fn occur_in(&self, body: &[u8]) -> bool { + if self.finders.is_empty() { + return self.occur_in_windows(body); + } + self.finders + .iter() + .any(|finder| finder.find(body).is_some()) + } + + /// [`Self::occur_in`] by a scan of every four-byte window of `body`. + fn occur_in_windows(&self, body: &[u8]) -> bool { body.windows(4).any(|window| { let value = u32::from_le_bytes([window[0], window[1], window[2], window[3]]); let (word, bit) = Self::slot(value); @@ -208,6 +234,71 @@ impl IdNeedles { }) } } + +#[cfg(test)] +mod needle_tests { + use super::{IdNeedles, MAX_FINDERS}; + + /// What `occur_in` and `occur_in_windows` say of `body` for `ids`. + fn both(ids: &[i64], body: &[u8]) -> (bool, bool) { + let needles = IdNeedles::new(ids.iter().copied()); + (needles.occur_in(body), needles.occur_in_windows(body)) + } + + #[test] + fn an_id_in_the_last_four_bytes_is_found() { + let mut body = vec![0xaa, 0xbb, 0xcc]; + body.extend_from_slice(&0x1234_5678u32.to_le_bytes()); + assert_eq!(both(&[0x1234_5678], &body), (true, true)); + } + + #[test] + fn an_id_at_an_unaligned_offset_is_found() { + let mut body = vec![0x01]; + body.extend_from_slice(&7u32.to_le_bytes()); + body.extend_from_slice(&[0x02, 0x03]); + assert_eq!(both(&[7], &body), (true, true)); + } + + #[test] + fn a_body_shorter_than_an_id_holds_none() { + assert_eq!(both(&[7], &[7, 0, 0]), (false, false)); + assert_eq!(both(&[7], &[]), (false, false)); + } + + #[test] + fn a_window_that_shares_only_the_low_sixteen_bits_is_not_an_id() { + let body = 0x0001_0007u32.to_le_bytes(); + assert_eq!(both(&[7], &body), (false, false)); + assert_eq!(both(&[0x0002_0007], &body), (false, false)); + } + + #[test] + fn no_ids_find_nothing() { + assert_eq!(both(&[], &7u32.to_le_bytes()), (false, false)); + } + + #[test] + fn an_eight_byte_id_is_found_by_its_first_four_bytes() { + let body = 0x0000_0001_0000_0009u64.to_le_bytes(); + assert_eq!(both(&[0x0000_0001_0000_0009], &body), (true, true)); + } + + #[test] + fn the_search_and_the_scan_agree_below_and_above_the_most_searched_for() { + let present = 0x00c0_ffee_i64; + let mut body = vec![0u8; 5]; + body.extend_from_slice(&(present as u32).to_le_bytes()); + body.extend_from_slice(&[9; 6]); + for count in [1, MAX_FINDERS, MAX_FINDERS + 1, 40] { + let mut ids: Vec = (0..count as i64 - 1).map(|n| 0x7000_0000 + n).collect(); + ids.push(present); + assert_eq!(both(&ids, &body), (true, true), "{count} ids"); + let absent: Vec = (0..count as i64).map(|n| 0x7000_0000 + n).collect(); + assert_eq!(both(&absent, &body), (false, false), "{count} absent ids"); + } + } +} #[derive(Debug, Serialize)] pub struct Record { pub identity: Identity,