diff --git a/changelog.d/changed/1335-definitions-projected.md b/changelog.d/changed/1335-definitions-projected.md new file mode 100644 index 000000000..93fa55a6e --- /dev/null +++ b/changelog.d/changed/1335-definitions-projected.md @@ -0,0 +1,8 @@ +- **The parameter definitions pass no longer builds the values of the + `Family` and `FamilySymbol` records it reads two fields of (#1335).** The + pass decoded each of those records in full and kept the record's `m_id` and + its `m_categoryId` or `m_familyId`, which was most of the pass on a Revit 2015 + to 2023 project. It now reads them as before, with the same checks and the + same refusals, and builds only the fields it keeps + (`native_parameters::decode_graph_projected`, `Options::graph_fields`). No + export changes. The figures are in the pull request. diff --git a/changelog.d/changed/1335-phase-projected.md b/changelog.d/changed/1335-phase-projected.md new file mode 100644 index 000000000..abc775776 --- /dev/null +++ b/changelog.d/changed/1335-phase-projected.md @@ -0,0 +1,5 @@ +- **The phase read no longer builds the values of the room fields it ignores + (#1335).** The read of the project's phases, and of the phase each room + names, decoded every `RoomElem` graph in full to read three fields of it. It + reads and checks the same bytes, with the same refusals, and builds only + those fields. No export changes. The figures are in the pull request. diff --git a/probes/examples/probe_1335_projected_graphs.rs b/probes/examples/probe_1335_projected_graphs.rs new file mode 100644 index 000000000..34775d195 --- /dev/null +++ b/probes/examples/probe_1335_projected_graphs.rs @@ -0,0 +1,276 @@ +//! #1335: a projected graph decode is the full decode with fewer values built. +//! +//! FACT under test: `native_parameters::decode_graph_projected(.., keep)` +//! succeeds exactly where `decode_graph_with_catalog_usage` does and fails +//! with the same error where that fails; where it succeeds the graph is the +//! full one's in everything but the values of the fields not in `keep`: the +//! same objects (class, token, start, end of fields), edges, consumed bytes and +//! usage, and each object's fields exactly the full object's entries whose key +//! is in `keep`. The definitions pass reads a `Family` record's `m_id` and +//! `m_categoryId` and a `FamilySymbol` record's `m_id` and `m_familyId` and +//! nothing else, so its binding context (`symbol_families`, +//! `family_categories`) and its refusals are the same read from either graph. +//! +//! Every record of the file's native partitions is decoded in full and +//! projected under each of `KEEPS`, whatever its class: the lists are the ones +//! the definitions pass, the wall type and top join read and the phase read use, +//! the identity alone, and the empty list (every field left out). +//! +//! Output, one line each: +//! file release +//! compared records decodes full-ok full-refused mismatches +//! budget-stopped +//! keep compared mismatches +//! refused (a full decode that failed, by class) +//! binding-context family-categories symbol-families same +//! ingest-refusals same-refusals +//! mismatch (the first few) +//! +//! Usage: +//! cargo run --profile ci -p rvt-probes --example probe_1335_projected_graphs -- FILE +use anyhow::Result; +use rvt::RevitFile; +use rvt::research::native_document::{self, Options}; +use rvt::research::native_es_catalog; +use rvt::research::native_parameter_definitions::Registry as Definitions; +use rvt::research::native_parameters::{ + GraphLimits, ObjectGraph, decode_graph_projected, decode_graph_with_catalog_usage, +}; +use rvt::research::schema_registry; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::{Duration, Instant}; + +const BUDGET: Duration = Duration::from_secs(12 * 60); +const SHOWN: usize = 12; + +const KEEPS: [&[&str]; 6] = [ + &["m_id", "m_categoryId"], + &["m_id", "m_familyId"], + &["m_id", "m_WallAttributesId", "m_function", "m_baseOrTop"], + &[ + "m_id", + "m_createdPhaseId", + "m_demolishedPhaseId", + "m_phaseId", + ], + &["m_id"], + &[], +]; + +fn filtered(fields: &Value, keep: &[&str]) -> Value { + match fields { + Value::Object(map) => Value::Object( + map.iter() + .filter(|(key, _)| keep.contains(&key.as_str())) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + ), + other => other.clone(), + } +} + +fn identifier(value: &Value) -> Option { + if let Some(n) = value.as_i64() { + return Some(n); + } + ["m_id", "m_id64"] + .iter() + .find_map(|key| value.get(*key)) + .and_then(identifier) +} + +/// What the definitions pass does with a Family or FamilySymbol graph: the +/// identity check of `extract_records`, then `ingest_binding_context`. +fn ingest(definitions: &mut Definitions, graph: &ObjectGraph, id: u64) -> Result<(), String> { + let root = graph.objects.first().ok_or("empty owner graph")?; + let saved = identifier(&root.fields["m_id"]).ok_or("unsupported serialized ElementId value")?; + if u64::try_from(saved).ok() != Some(id) { + return Err(format!( + "serialized owner identity disagrees with indexed record {id}" + )); + } + definitions + .ingest_binding_context(graph) + .map_err(|error| error.to_string()) +} + +fn differences(full: &ObjectGraph, projected: &ObjectGraph, keep: &[&str]) -> Vec<&'static str> { + let mut found = Vec::new(); + if full.consumed_bytes != projected.consumed_bytes { + found.push("consumed"); + } + if full.objects.len() != projected.objects.len() { + found.push("object-count"); + return found; + } + for (a, b) in full.objects.iter().zip(&projected.objects) { + if a.class_tag != b.class_tag + || a.class_name != b.class_name + || a.token != b.token + || a.start != b.start + || a.fields_end != b.fields_end + { + found.push("object"); + } + let expected = if a.class_name == "ExtensibleStorageEntity" { + a.fields.clone() + } else { + filtered(&a.fields, keep) + }; + if expected != b.fields { + found.push("fields"); + } + } + let same_edges = full.edges.len() == projected.edges.len() + && full.edges.iter().zip(&projected.edges).all(|(a, b)| { + a.source_object_index == b.source_object_index + && a.pointer_offset == b.pointer_offset + && a.pointer_token == b.pointer_token + && a.target_object_index == b.target_object_index + && a.target_class_tag == b.target_class_tag + }); + if !same_edges { + found.push("edges"); + } + found +} + +fn main() -> Result<()> { + let path = std::env::args() + .nth(1) + .ok_or_else(|| anyhow::anyhow!("usage: probe_1335_projected_graphs FILE"))?; + let mut rf = RevitFile::open(&path)?; + let release = rf.layout_release()?; + println!("file {path} release {release}"); + if !native_document::NATIVE_RELEASES.contains(&release) { + println!("native: release {release} is outside the native readers' range"); + return Ok(()); + } + let formats = native_document::read_single(&mut rf, "Formats/Latest")?; + let registry = schema_registry::parse(&formats)?; + let catalog = native_document::read_single(&mut rf, "Global/Latest") + .and_then(|bytes| native_es_catalog::decode(&bytes, ®istry)) + .ok() + .map(|(catalog, _)| catalog); + let limits = GraphLimits { + max_values: 100_000, + max_objects: 100_000, + ..GraphLimits::default() + }; + // Every record is handed over with its body and no graph: this probe + // decodes them itself. + let options = Options { + releases: native_document::NATIVE_RELEASES, + graph_classes: Some(BTreeSet::new()), + keep_body: true, + ..Options::default() + }; + let started = Instant::now(); + let mut records = 0usize; + let mut decodes = 0usize; + let mut full_ok = 0usize; + let mut full_refused = 0usize; + let mut mismatches = 0usize; + let mut stopped = false; + let mut per_keep = [(0usize, 0usize); KEEPS.len()]; + let mut refused: BTreeMap = BTreeMap::new(); + let mut full_definitions = Definitions::default(); + let mut projected_definitions = Definitions::default(); + let mut ingest_refusals = 0usize; + let mut same_refusals = true; + let mut shown = 0usize; + let result = native_document::extract_graphs(&mut rf, &options, |record| { + if started.elapsed() > BUDGET { + stopped = true; + anyhow::bail!("time budget after {records} records"); + } + let Some(body) = &record.body else { + return Ok(()); + }; + records += 1; + let class = record.class_name.clone().unwrap_or_default(); + let full = decode_graph_with_catalog_usage(body, ®istry, &limits, catalog.as_ref()); + match &full { + Ok(_) => full_ok += 1, + Err(_) => { + full_refused += 1; + *refused.entry(class.clone()).or_default() += 1; + } + } + for (index, keep) in KEEPS.iter().enumerate() { + let projected = + decode_graph_projected(body, ®istry, &limits, catalog.as_ref(), keep); + decodes += 1; + per_keep[index].0 += 1; + let kind = match (&full, &projected) { + (Ok((graph, usage)), Ok((other, other_usage))) => { + if usage.values != other_usage.values || usage.objects != other_usage.objects { + Some("usage".to_string()) + } else { + differences(graph, other, keep) + .first() + .map(|k| k.to_string()) + } + } + (Err(a), Err(b)) => (a.to_string() != b.to_string()).then(|| "error".to_string()), + (Ok(_), Err(_)) => Some("projected-refused".to_string()), + (Err(_), Ok(_)) => Some("projected-accepted".to_string()), + }; + if let Some(kind) = kind { + mismatches += 1; + per_keep[index].1 += 1; + if shown < SHOWN { + shown += 1; + println!("mismatch {kind} {class}"); + } + } + } + if matches!(class.as_str(), "Family" | "FamilySymbol") { + let keep: &[&str] = if class == "Family" { + KEEPS[0] + } else { + KEEPS[1] + }; + let projected = + decode_graph_projected(body, ®istry, &limits, catalog.as_ref(), keep); + let id = record.identity.element_id; + let a = match &full { + Ok((graph, _)) => ingest(&mut full_definitions, graph, id), + Err(error) => Err(error.to_string()), + }; + let b = match &projected { + Ok((graph, _)) => ingest(&mut projected_definitions, graph, id), + Err(error) => Err(error.to_string()), + }; + ingest_refusals += usize::from(a.is_err()); + same_refusals &= a == b; + } + Ok(()) + }); + if let Err(error) = &result { + if !stopped { + println!("native: {error:#}"); + } + } + println!( + "compared records {records} decodes {decodes} full-ok {full_ok} full-refused {full_refused} mismatches {mismatches} budget-stopped {stopped}" + ); + for (keep, (compared, wrong)) in KEEPS.iter().zip(per_keep) { + println!( + "keep {} compared {compared} mismatches {wrong}", + keep.join(",") + ); + } + for (class, n) in &refused { + println!("refused {class} {n}"); + } + let same = full_definitions.family_categories == projected_definitions.family_categories + && full_definitions.symbol_families == projected_definitions.symbol_families; + println!( + "binding-context family-categories {} symbol-families {} same {same} ingest-refusals {ingest_refusals} same-refusals {same_refusals}", + full_definitions.family_categories.len(), + full_definitions.symbol_families.len(), + ); + Ok(()) +} diff --git a/src/research/native_document.rs b/src/research/native_document.rs index 5276e9e7e..1225731a6 100644 --- a/src/research/native_document.rs +++ b/src/research/native_document.rs @@ -25,6 +25,10 @@ pub struct ResourceBudget { pub max_graph_objects: usize, } +/// The fields to keep of a record's graph, by its root class +/// ([`Options::graph_fields`]). +pub type GraphFields = fn(&str) -> Option<&'static [&'static str]>; + #[derive(Debug, Clone)] pub struct Options { pub selected_ids: BTreeSet, @@ -95,6 +99,19 @@ pub struct Options { /// RE-342, which reads saved graphics on a file of Revit 2015 to 2022 with /// [`Self::releases`] widened, turns it on. pub deferred_history_episodes: bool, + /// For a record whose root class this names, the fields of its graph's + /// objects that are kept: those whose key (a bare field name, or + /// `Declaring::name` where the class chain declares it more than once) is + /// in the list, and no other ([`native_parameters::decode_graph_projected`]). + /// The record is read and checked as a full one is, so it is refused for + /// the same reasons, but the values of the fields left out are never built. + /// The list names `m_id` as well, which the check of a record against its + /// identity reads. A projected record has no [`Record::saved_metadata`], + /// which is made from the whole graph. A class the function gives `None` + /// for, and any record with no function here, keeps every field. For a + /// reader that takes only those fields from the records of those classes + /// (#1335). + pub graph_fields: Option, } /// The default [`Options::max_group_bytes`]: one reassembled continuation /// group's bytes. @@ -146,6 +163,7 @@ impl Default for Options { keep_body: false, value_source: None, deferred_history_episodes: false, + graph_fields: None, } } } @@ -574,6 +592,19 @@ pub(crate) type DefinitionKey = ( bool, ); +/// The fields of a `Family` and a `FamilySymbol` record that +/// [`crate::native_parameter_definitions::Registry::ingest_binding_context`] +/// reads, with `m_id`, which the record's identity is checked against: the +/// definitions pass takes nothing else from them (#1335). The rest of those +/// records is most of the pass. +fn binding_context_fields(class: &str) -> Option<&'static [&'static str]> { + match class { + "Family" => Some(&["m_id", "m_categoryId"]), + "FamilySymbol" => Some(&["m_id", "m_familyId"]), + _ => None, + } +} + /// The handle's [`DefinitionPass`] for `options`, read if it has none. fn definition_pass(file: &mut RevitFile, options: &Options) -> Result> { // Every field is named, so a new one has to be put in the key or left out @@ -594,6 +625,7 @@ fn definition_pass(file: &mut RevitFile, options: &Options) -> Result Result native_parameters::decode_graph_projected( + body, + registry, + &limits, + summary.extensible_storage_catalog.as_ref(), + keep, + ), + None => native_parameters::decode_graph_with_catalog_usage( + body, + registry, + &limits, + summary.extensible_storage_catalog.as_ref(), + ), + }; let decoded = decoded.and_then(|(graph, usage)| { if source.channel == 102 { let root = graph @@ -1184,13 +1230,14 @@ fn extract_records( .entry(diagnostic.clone().unwrap_or_default()) .or_default() += 1; } - let (saved_metadata, metadata_diagnostic) = match graph.as_ref().map(|graph| { - crate::native_metadata::project_with_definitions(graph, definitions) - }) { - Some(Ok(value)) => (Some(value), None), - Some(Err(error)) => (None, Some(error.to_string())), - None => (None, None), - }; + let (saved_metadata, metadata_diagnostic) = + match graph.as_ref().filter(|_| projected.is_none()).map(|graph| { + crate::native_metadata::project_with_definitions(graph, definitions) + }) { + Some(Ok(value)) => (Some(value), None), + Some(Err(error)) => (None, Some(error.to_string())), + None => (None, None), + }; summary.projected_metadata_records += usize::from(saved_metadata.is_some()); summary.unsupported_metadata_records += usize::from(metadata_diagnostic.is_some()); diff --git a/src/research/native_parameters.rs b/src/research/native_parameters.rs index 08ada846e..084889248 100644 --- a/src/research/native_parameters.rs +++ b/src/research/native_parameters.rs @@ -155,6 +155,35 @@ pub fn decode_graph_with_catalog_usage( registry: &Registry, limits: &GraphLimits, catalog: Option<&crate::native_extensible_storage::Catalog>, +) -> Result<(ObjectGraph, GraphUsage)> { + decode_graph_in(body, registry, limits, catalog, None) +} +/// [`decode_graph_with_catalog_usage`] keeping only the fields of every object +/// whose key is in `keep` (a bare field name, or `Declaring::name` where the +/// class chain declares the name more than once). Any other field is read and +/// checked as the whole decode reads and checks it, and left out of the +/// object's `fields`, which then hold exactly the entries of `keep` that the +/// full decode's do, with the same values. The graph is otherwise the full +/// decode's: it succeeds or fails with the same error, reads the same pointers +/// and objects, and has the same object starts and ends, edges, consumed bytes +/// and usage. A caller that reads only the fields in `keep` of a graph gets +/// what it gets from the full decode, without the values it ignores being +/// built and freed (#1335). +pub fn decode_graph_projected( + body: &[u8], + registry: &Registry, + limits: &GraphLimits, + catalog: Option<&crate::native_extensible_storage::Catalog>, + keep: &[&str], +) -> Result<(ObjectGraph, GraphUsage)> { + decode_graph_in(body, registry, limits, catalog, Some(keep)) +} +fn decode_graph_in( + body: &[u8], + registry: &Registry, + limits: &GraphLimits, + catalog: Option<&crate::native_extensible_storage::Catalog>, + keep: Option<&[&str]>, ) -> Result<(ObjectGraph, GraphUsage)> { ensure!( limits.max_values > 0 && limits.max_objects > 0 && limits.max_depth > 0, @@ -162,7 +191,7 @@ pub fn decode_graph_with_catalog_usage( ); let tag = le_bytes::u16_at(body, 0).ok_or_else(|| anyhow::anyhow!("truncated root object tag"))?; - let mut cursor = Cursor::new(body, registry, catalog, 2, *limits, Spans::None); + let mut cursor = Cursor::new(body, registry, catalog, 2, *limits, Spans::None).with_keep(keep); let mut objects = Vec::new(); let mut edges = Vec::new(); let mut shared = BTreeMap::::new(); @@ -186,7 +215,7 @@ pub fn decode_graph_with_catalog_usage( .name .clone(); let fields = cursor - .class(tag, 0) + .class_in::(tag, 0, true) .map_err(|e| anyhow::anyhow!("graph {name} starts{start}: {e}"))?; (name, fields) }; @@ -310,7 +339,7 @@ pub fn decode_object_fields_spanning( .name .clone(); let mut cursor = Cursor::new(body, registry, None, start, *limits, spans); - let fields = cursor.class(class_tag, 0)?; + let fields = cursor.class::(class_tag, 0)?; Ok(ObjectFields { class_tag, class_name, @@ -334,6 +363,15 @@ struct PendingPointer { struct ClassPlan<'a> { lineage: Vec<&'a Class>, keys: Vec>, + /// Whether each field is one a projected decode keeps + /// ([`decode_graph_projected`]); empty for a decode that is not one. + kept: Vec>, + /// The position, as (index in `lineage`, index in the class's fields), of + /// the first field whose key an earlier one in decoding order has already + /// taken: where a decode that builds no object stops with the error a + /// built one meets there. Found only for a projected decode; `None` + /// otherwise. + duplicate: Option<(usize, usize)>, } struct Cursor<'a> { body: &'a [u8], @@ -346,6 +384,8 @@ struct Cursor<'a> { keep_spans: Spans, plans: HashMap>>, limits: GraphLimits, + /// The fields a projected decode keeps ([`decode_graph_projected`]). + keep: Option<&'a [&'a str]>, } impl<'a> Cursor<'a> { fn new( @@ -367,8 +407,13 @@ impl<'a> Cursor<'a> { keep_spans, plans: HashMap::new(), limits, + keep: None, } } + fn with_keep(mut self, keep: Option<&'a [&'a str]>) -> Self { + self.keep = keep; + self + } fn take(&mut self, n: usize) -> Result<&[u8]> { let end = self .pos @@ -384,7 +429,7 @@ impl<'a> Cursor<'a> { fn u32(&mut self) -> Result { Ok(u32::from_le_bytes(self.take(4)?.try_into()?)) } - fn pointer(&mut self, external: bool) -> Result { + fn pointer(&mut self, external: bool) -> Result { let offset = self.pos; let token = self.u32()?; let tag = if token != 0 && !external { @@ -403,9 +448,13 @@ impl<'a> Cursor<'a> { } else { None }; - Ok(json!({"pointer_token":token,"class_tag":tag,"offset":offset})) + Ok(if BUILD { + json!({"pointer_token":token,"class_tag":tag,"offset":offset}) + } else { + Value::Null + }) } - fn entity_pointer(&mut self) -> Result { + fn entity_pointer(&mut self) -> Result { let offset = self.pos; let discriminator = i32::from_le_bytes(self.take(4)?.try_into()?); ensure!( @@ -413,7 +462,11 @@ impl<'a> Cursor<'a> { "unqualified ES entity discriminator {discriminator}" ); if discriminator == 0 { - return Ok(json!({"discriminator":0,"state":"serialized_absent","offset":offset})); + return Ok(if BUILD { + json!({"discriminator":0,"state":"serialized_absent","offset":offset}) + } else { + Value::Null + }); } let raw: [u8; 16] = self.take(16)?.try_into()?; let guid = crate::native_extensible_storage::guid_string(raw); @@ -427,9 +480,11 @@ impl<'a> Cursor<'a> { offset, schema_guid: Some(guid.clone()), }); - Ok( - json!({"discriminator":-1,"schema_guid":guid,"state":"serialized_present","offset":offset}), - ) + Ok(if BUILD { + json!({"discriminator":-1,"schema_guid":guid,"state":"serialized_present","offset":offset}) + } else { + Value::Null + }) } fn entity_fields(&mut self, guid: &str) -> Result { let schema = self @@ -503,7 +558,7 @@ impl<'a> Cursor<'a> { } modifier = 0x50; } - self.field( + self.field::( &Field { offset: 0, name: name.into(), @@ -520,12 +575,22 @@ impl<'a> Cursor<'a> { 0, ) } - fn class(&mut self, tag: u16, depth: usize) -> Result { + /// The object of class `tag` at the cursor. With `BUILD` unset nothing is + /// built: every byte is read, every check made and every pointer noted as + /// when it is set, and the value is `Null` (#1335). + fn class(&mut self, tag: u16, depth: usize) -> Result { + self.class_in::(tag, depth, false) + } + /// [`Self::class`], where `top` says the object is one the graph decode + /// itself starts on, not an inline one inside a field: only then are the + /// fields chosen by [`Self::keep`] the only ones built. + fn class_in(&mut self, tag: u16, depth: usize, top: bool) -> Result { ensure!( depth < self.limits.max_depth, "native class recursion budget exceeded" ); let plan = self.plan(tag, depth)?; + let project = BUILD && top && self.keep.is_some(); let mut values = Map::new(); for (distance, c) in plan.lineage.iter().enumerate().rev() { for (field_index, f) in c.fields.iter().enumerate() { @@ -546,33 +611,11 @@ impl<'a> Cursor<'a> { f.uninterpreted_flags ); let start = self.pos; - let value = if c.name == "ClassDefinitionRef" - && f.name == "m_ref" - && f.base == 10 - && f.modifier == 0 - { - // ClassDefinitionRef stores a 16-bit schema class tag. The - // qualified external reader performs Int16Reader -> ClassesContainer.find. - let class_tag = u16::from_le_bytes(self.take(2)?.try_into()?); - json!({"class_tag":class_tag,"class_name":self.registry.class(class_tag).map(|class|class.name.as_str())}) - } else if c.name == "ESEntity" - && f.name == "m_blob" - && f.base == 14 - && f.modifier == 1 - { - self.entity_pointer()? - } else if c.name == "VarSketchObj" - && f.name == "m_params" - && f.base == 14 - && f.modifier == 0x54 - { - // Qualified typeless owning-pointer vector: each entry still - // carries its concrete VarParam class tag in the wire data. - self.field_as(f, 0x51, f.array_count, depth + distance + 1)? + let kept = BUILD && (!project || plan.kept[distance][field_index]); + let value = if kept { + self.class_field::(c, f, depth, distance)? } else { - self.field(f, depth + distance + 1).map_err(|e| { - anyhow::anyhow!("{}.{} at {}: {e}", c.name, f.name, self.pos) - })? + self.class_field::(c, f, depth, distance)? }; if self.keeps_spans_of(c.tag) { self.spans.push(FieldSpan { @@ -582,17 +625,64 @@ impl<'a> Cursor<'a> { end: self.pos, }); } - let key = plan.keys[distance][field_index].clone(); - ensure!( - !values.contains_key(&key), - "ambiguous native field key {key} in declaring class {} (tag {})", - c.name, - c.tag - ); - values.insert(key, value); + if kept { + let key = plan.keys[distance][field_index].clone(); + ensure!( + !values.contains_key(&key), + "ambiguous native field key {key} in declaring class {} (tag {})", + c.name, + c.tag + ); + values.insert(key, value); + } else { + ensure!( + plan.duplicate != Some((distance, field_index)), + "ambiguous native field key {} in declaring class {} (tag {})", + plan.keys[distance][field_index], + c.name, + c.tag + ); + } } } - Ok(Value::Object(values)) + Ok(if BUILD { + Value::Object(values) + } else { + Value::Null + }) + } + /// One field `f` of class `c`, the `distance`th of the chain from the + /// object's own class, as [`Self::class_in`] reads it. + fn class_field( + &mut self, + c: &Class, + f: &Field, + depth: usize, + distance: usize, + ) -> Result { + if c.name == "ClassDefinitionRef" && f.name == "m_ref" && f.base == 10 && f.modifier == 0 { + // ClassDefinitionRef stores a 16-bit schema class tag. The + // qualified external reader performs Int16Reader -> ClassesContainer.find. + let class_tag = u16::from_le_bytes(self.take(2)?.try_into()?); + Ok(if BUILD { + json!({"class_tag":class_tag,"class_name":self.registry.class(class_tag).map(|class|class.name.as_str())}) + } else { + Value::Null + }) + } else if c.name == "ESEntity" && f.name == "m_blob" && f.base == 14 && f.modifier == 1 { + self.entity_pointer::() + } else if c.name == "VarSketchObj" + && f.name == "m_params" + && f.base == 14 + && f.modifier == 0x54 + { + // Qualified typeless owning-pointer vector: each entry still + // carries its concrete VarParam class tag in the wire data. + self.field_as::(f, 0x51, f.array_count, depth + distance + 1) + } else { + self.field::(f, depth + distance + 1) + .map_err(|e| anyhow::anyhow!("{}.{} at {}: {e}", c.name, f.name, self.pos)) + } } /// Whether the spans of the fields class `tag` declares are kept. fn keeps_spans_of(&self, tag: u16) -> bool { @@ -662,18 +752,59 @@ impl<'a> Cursor<'a> { .collect::>() }) .collect(); - let plan = Rc::new(ClassPlan { lineage, keys }); + let kept = match self.keep { + Some(keep) => lineage + .iter() + .zip(&keys) + .map(|(c, keys)| { + c.fields + .iter() + .zip(keys) + .map(|(f, key)| { + f.uninterpreted_flags & 2 == 0 && keep.contains(&key.as_str()) + }) + .collect::>() + }) + .collect(), + None => Vec::new(), + }; + // Only a projected decode reads fields without building them, and so + // needs the position; every other decode finds a repeated key in the + // object it builds. + let duplicate = if self.keep.is_some() { + let mut taken = std::collections::HashSet::new(); + let mut duplicate = None; + 'chain: for (distance, c) in lineage.iter().enumerate().rev() { + for (field_index, f) in c.fields.iter().enumerate() { + if f.uninterpreted_flags & 2 == 0 + && !taken.insert(keys[distance][field_index].as_str()) + { + duplicate = Some((distance, field_index)); + break 'chain; + } + } + } + duplicate + } else { + None + }; + let plan = Rc::new(ClassPlan { + lineage, + keys, + kept, + duplicate, + }); self.plans.insert(tag, Rc::clone(&plan)); Ok(plan) } - fn field(&mut self, f: &Field, depth: usize) -> Result { - self.field_as(f, f.modifier, f.array_count, depth) + fn field(&mut self, f: &Field, depth: usize) -> Result { + self.field_as::(f, f.modifier, f.array_count, depth) } /// [`Self::field`] reading `f` with `modifier` and `array_count` in place /// of its own: a container's items are `f` itself, one level down, which /// copying the field for each container was a share of a graph's /// allocations (#1237). - fn field_as( + fn field_as( &mut self, f: &Field, modifier: u8, @@ -692,12 +823,21 @@ impl<'a> Cursor<'a> { ); self.items += 1; match modifier { - 1..=3 if f.base == 14 => self.pointer(modifier == 3), + 1..=3 if f.base == 14 => self.pointer::(modifier == 3), 0x60 if f.base == 8 => { let n = self.u32()? as usize; ensure!(n <= 1_000_000, "native string unit budget exceeded"); - let units = le_bytes::u16_units(self.take(n * 2)?).collect::>(); - Ok(Value::String(String::from_utf16(&units)?)) + let bytes = self.take(n * 2)?; + if BUILD { + let units = le_bytes::u16_units(bytes).collect::>(); + Ok(Value::String(String::from_utf16(&units)?)) + } else { + if char::decode_utf16(le_bytes::u16_units(bytes)).any(|unit| unit.is_err()) { + let units = le_bytes::u16_units(bytes).collect::>(); + String::from_utf16(&units)?; + } + Ok(Value::Null) + } } 0x10 | 0x11 | 0x12 | 0x13 | 0x50 | 0x51 | 0x52 | 0x53 => { let n = if modifier < 0x50 { @@ -706,35 +846,64 @@ impl<'a> Cursor<'a> { self.u32()? }; ensure!(n <= 100_000, "native container count budget exceeded"); - let mut values = Vec::with_capacity(n.min(1024) as usize); - for _ in 0..n { - values.push(self.field_as(f, modifier & 0xf, None, depth + 1)?); + if BUILD { + let mut values = Vec::with_capacity(n.min(1024) as usize); + for _ in 0..n { + values.push(self.field_as::(f, modifier & 0xf, None, depth + 1)?); + } + Ok(Value::Array(values)) + } else { + for _ in 0..n { + self.field_as::(f, modifier & 0xf, None, depth + 1)?; + } + Ok(Value::Null) } - Ok(Value::Array(values)) } 0 => match f.base { 1 => { let b = self.take(1)?[0]; ensure!(b <= 1, "invalid native bool"); - Ok(json!(b != 0)) + Ok(if BUILD { json!(b != 0) } else { Value::Null }) + } + 2 => { + let b = self.take(1)?[0]; + Ok(if BUILD { json!(b) } else { Value::Null }) + } + 3 => { + let n = i16::from_le_bytes(self.take(2)?.try_into()?); + Ok(if BUILD { json!(n) } else { Value::Null }) + } + 4 => { + let n = i32::from_le_bytes(self.take(4)?.try_into()?); + Ok(if BUILD { json!(n) } else { Value::Null }) + } + 5 => { + let n = self.u32()?; + Ok(if BUILD { json!(n) } else { Value::Null }) } - 2 => Ok(json!(self.take(1)?[0])), - 3 => Ok(json!(i16::from_le_bytes(self.take(2)?.try_into()?))), - 4 => Ok(json!(i32::from_le_bytes(self.take(4)?.try_into()?))), - 5 => Ok(json!(self.u32()?)), 6 => { let n = f32::from_le_bytes(self.take(4)?.try_into()?); ensure!(n.is_finite(), "nonfinite native float"); - Ok(json!(n)) + Ok(if BUILD { json!(n) } else { Value::Null }) } 7 => { let n = f64::from_le_bytes(self.take(8)?.try_into()?); ensure!(n.is_finite(), "nonfinite native double"); - Ok(json!(n)) + Ok(if BUILD { json!(n) } else { Value::Null }) + } + 9 => { + let bytes = self.take(16)?; + Ok(if BUILD { + json!({"guid_bytes":bytes}) + } else { + Value::Null + }) + } + 11 => { + let n = i64::from_le_bytes(self.take(8)?.try_into()?); + Ok(if BUILD { json!(n) } else { Value::Null }) } - 9 => Ok(json!({"guid_bytes":self.take(16)?})), - 11 => Ok(json!(i64::from_le_bytes(self.take(8)?.try_into()?))), - 13 => self.field( + 13 => self.field::( f.nested_descriptor .as_deref() .ok_or_else(|| anyhow::anyhow!("nested schema absent"))?, @@ -745,7 +914,7 @@ impl<'a> Cursor<'a> { f.references.len() == 1, "inline native reference schema count" ); - self.class(f.references[0].tag, depth + 1) + self.class::(f.references[0].tag, depth + 1) } _ => anyhow::bail!("unsupported native scalar {}", f.base), }, @@ -766,7 +935,7 @@ fn identifier(value: &Value) -> Result { pub fn decode(body: &[u8], registry: &Registry) -> Result { let base = crate::native_element::decode_base(body, registry)?; let mut cursor = Cursor::new(body, registry, None, 2, GraphLimits::default(), Spans::None); - let fields = cursor.class(base.class_tag, 0)?; + let fields = cursor.class::(base.class_tag, 0)?; let derived_fields_end = cursor.pos; let mut parameters = Vec::new(); for (name, class_name, storage) in [ @@ -797,7 +966,7 @@ pub fn decode(body: &[u8], registry: &Registry) -> Result { registry.class(tag).is_some_and(|c| c.name == class_name), "parameter set class mismatch" ); - let set = cursor.class(tag, 0)?; + let set = cursor.class::(tag, 0)?; let pairs = set["m_paramSet"] .as_array() .ok_or_else(|| anyhow::anyhow!("parameter pair container absent"))?; diff --git a/src/research/native_phases.rs b/src/research/native_phases.rs index 998020259..2b5ceec5b 100644 --- a/src/research/native_phases.rs +++ b/src/research/native_phases.rs @@ -372,6 +372,23 @@ pub fn exclusion( .then_some(PhaseExclusion::Demolished) } +/// The fields of a `RoomElem` record that [`compute_phase_filter`] takes: the +/// phases it was created and demolished in, its own phase, and `m_id`, which +/// the record's identity is checked against. The rest of a room's graph is +/// read and checked but not built (#1335). The two records of the project's +/// phases are read in full. +fn room_phase_fields(class: &str) -> Option<&'static [&'static str]> { + match class { + "RoomElem" => Some(&[ + "m_id", + "m_createdPhaseId", + "m_demolishedPhaseId", + "m_phaseId", + ]), + _ => None, + } +} + /// The project's phase order and names, every element not in its last phase /// and every room's own phase, in one pass over the native records (graphs /// only for `AllProjectPhases`, `ProjectPhase` and `RoomElem`). Empty where @@ -392,6 +409,7 @@ pub fn compute_phase_filter(rf: &mut RevitFile) -> anyhow::Result { "RoomElem".to_string(), ])), releases: native_document::NATIVE_RELEASES, + graph_fields: Some(room_phase_fields), ..native_document::Options::default() }; let mut order: Option> = None;