From a218f0c4435bcabb200570b87a957e35524ff5df Mon Sep 17 00:00:00 2001 From: mavrick Date: Wed, 4 Nov 2026 10:38:28 -0600 Subject: [PATCH 1/4] fix(security): keep credentials within runtime boundaries Classify supported address fields consistently, validate service forwarding, and report counted removals through rollback-safe notices. Prepare durable output transactionally and bound runtime restoration for saved/widget views. --- SECURITY.md | 12 + apps/rustgis-python/python/rustgis/_embed.py | 14 +- apps/rustgis-python/src/lib.rs | 33 +- apps/rustgis-python/tests/test_widget.py | 43 ++ apps/rustgis-web/src/embed.rs | 5 + apps/rustgis-web/src/web.rs | 11 +- crates/assistant/src/agent.rs | 15 + crates/assistant/src/audit.rs | 136 ++-- crates/assistant/src/tests.rs | 18 + crates/assistant/src/tools.rs | 14 +- crates/data/src/address.rs | 676 ++++++++++++++++++ crates/data/src/diagnostics.rs | 70 +- crates/data/src/lib.rs | 1 + crates/data/src/ogc.rs | 132 ++-- crates/data/src/source/credentials.rs | 98 +-- crates/data/src/source/pgconn.rs | 50 +- crates/engine/src/cmd/gp.rs | 9 +- crates/engine/src/cmd/history.rs | 110 +-- crates/engine/src/cmd/items.rs | 66 +- crates/engine/src/cmd/notebook.rs | 62 +- crates/engine/src/cmd/presentation.rs | 63 +- crates/engine/src/cmd/project.rs | 9 +- crates/engine/src/cmd/service.rs | 40 +- crates/engine/src/cmd/sharing.rs | 39 +- crates/engine/src/cmd/workflows.rs | 21 +- crates/engine/src/durable.rs | 700 +++++++++++++++++++ crates/engine/src/lib.rs | 127 +++- crates/engine/src/parameters.rs | 141 ++++ crates/engine/src/peer.rs | 300 ++++++++ crates/engine/src/project.rs | 366 ++++++---- crates/engine/src/project_log.rs | 28 +- crates/engine/src/roles.rs | 374 ++++++++++ crates/engine/src/sources.rs | 48 +- crates/engine/src/tests.rs | 161 +++-- crates/engine/src/tests/credential_output.rs | 315 +++++++++ crates/map/src/lib.rs | 206 ++++++ crates/map/src/raster.rs | 37 + crates/map/src/slots.rs | 46 ++ crates/map/src/source.rs | 68 ++ crates/map/src/tiles.rs | 58 ++ crates/tiles/src/arcgis_auth.rs | 2 +- crates/tiles/src/cache.rs | 6 +- crates/ui-egui/src/assistantpane.rs | 11 + crates/ui-egui/src/connectiontree.rs | 12 +- crates/ui-egui/src/lib.rs | 102 ++- crates/ui-egui/src/serviceui.rs | 11 +- docs/contributing.md | 8 + docs/mcp.md | 6 +- docs/python.md | 14 +- docs/user-guide/data.md | 18 +- docs/user-guide/web-data.md | 8 + docs/user-guide/workspace.md | 9 +- 52 files changed, 4246 insertions(+), 683 deletions(-) create mode 100644 crates/data/src/address.rs create mode 100644 crates/engine/src/durable.rs create mode 100644 crates/engine/src/parameters.rs create mode 100644 crates/engine/src/peer.rs create mode 100644 crates/engine/src/roles.rs create mode 100644 crates/engine/src/tests/credential_output.rs create mode 100644 crates/map/src/slots.rs diff --git a/SECURITY.md b/SECURITY.md index 78792ab8f..a371f5207 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,6 +19,18 @@ RustGIS is released often and fixes go into the next release; older releases don patches. Check that the problem still happens with the [latest release](https://github.com/opengeos/RustGIS/releases/latest) or `main`. +## Sharing projects and generated items + +Saving removes credentials from supported address fields and recognized executable parameters, +and reports counted removals. Feature data, user notes, foreign-language notebook code and opaque +plugin content can still contain sensitive information. An unchecked-content notice means that +content was preserved, not verified safe. Review it before sharing; saving is not a general +secret scanner. + +For an older file or log that exposed a key, revoke or rotate it with its service. Saving again +cleans only recognized fields and cannot recall shared copies. For a Jupyter notebook, also clear +saved widget state and save again; printed cell output and user data require your own review. + ## What counts as a security problem RustGIS makes promises about what untrusted plugins, files, agents and web pages can do. A way diff --git a/apps/rustgis-python/python/rustgis/_embed.py b/apps/rustgis-python/python/rustgis/_embed.py index 5f79c7af1..0d15bdd3b 100644 --- a/apps/rustgis-python/python/rustgis/_embed.py +++ b/apps/rustgis-python/python/rustgis/_embed.py @@ -67,6 +67,7 @@ def _start( # Notices waiting to be read. The app's messages may arrive on another thread (Jupyter's). self._held: List[Tuple[str, bool]] = [] self._held_lock = threading.Lock() + self._sent_keys = 0 if sample: self._session.run("project.sample") if data is not None: @@ -201,7 +202,11 @@ def to_png( def _push(self) -> None: """Sends the session's project to the app, when it changed.""" try: - text = self._session.project_json() + text, keys, notices = self._session._engine._peer_text() + self._hold([(notice, False) for notice in notices]) + if keys > self._sent_keys: + self._hold([(f"{keys} layer address(es) went to the map widget without their keys: those layers stay empty there", False)]) + self._sent_keys = keys except RustGISError as e: self._hold([(f"couldn't show the project in the app: {e}", True)]) return @@ -232,16 +237,15 @@ def _from_app(self, content: Any) -> None: text = content.get("text") if not isinstance(text, str) or text == self._project: return - # Opening it reports "Opened ...": not news here, so it is dropped, and what was - # waiting to be read is kept. + # Peer input is an import: keep its containment notices and invalidate old trust. self._hold(self._session.notices()) try: - self._session.run("project.openText", text=text) + self._session._engine._open_peer_text(text) except RustGISError as e: self._hold([(f"the app's project couldn't be read: {e}", True)]) return finally: - self._session.notices() + self._hold(self._session.notices()) # The app's own text, not the session's re-serialised copy: the app then sees its # project and doesn't open it again. self._project = text diff --git a/apps/rustgis-python/src/lib.rs b/apps/rustgis-python/src/lib.rs index 648862772..e8600a6d0 100644 --- a/apps/rustgis-python/src/lib.rs +++ b/apps/rustgis-python/src/lib.rs @@ -179,7 +179,38 @@ impl Engine { /// The project as `.rgis` JSON text, as `project.saveAs` writes it (what `project.openText` /// opens). fn project_json(&self, py: Python<'_>) -> PyResult { - let result = self.with_session(py, |session| serde_json::to_string(&session.project).map_err(|e| e.to_string())); + let result = self.with_session(py, |session| { + let (text, report) = session.project.saved_text(None)?; + for notice in report.notices() { + session.notify(notice, false); + } + Ok::<_, String>(text) + }); + self.flush(py); + result.map_err(RustGISError::new_err) + } + + /// Private bridge output: never remembered, and never includes keys in widget state. + fn _peer_text(&self, py: Python<'_>) -> PyResult<(String, u32, Vec)> { + let result = self.with_session(py, |session| { + let (text, report) = session.peer_text(rustgis_engine::Peer::Bridge)?; + let addresses = report.layer_addresses(); + let notices = report.notices().into_iter().skip(usize::from(addresses > 0)).collect(); + Ok::<_, String>((text, addresses, notices)) + }); + self.flush(py); + result.map_err(RustGISError::new_err) + } + + /// Private bridge ingress bypasses the saved-text cache and creates no User authority. + fn _open_peer_text(&self, py: Python<'_>, text: String) -> PyResult<()> { + let result = self.with_session(py, |session| { + let report = session.open_peer_text(rustgis_engine::Peer::Bridge, &text)?; + for notice in report.notices() { + session.notify(notice, false); + } + Ok::<_, String>(()) + }); self.flush(py); result.map_err(RustGISError::new_err) } diff --git a/apps/rustgis-python/tests/test_widget.py b/apps/rustgis-python/tests/test_widget.py index 888ef3e42..cd4194335 100644 --- a/apps/rustgis-python/tests/test_widget.py +++ b/apps/rustgis-python/tests/test_widget.py @@ -98,3 +98,46 @@ def test_layers_of_a_map_change_the_app(): sites.rename("Places") assert m._project != before and "Places" in m._project assert m.layer_names() == ["Places"] + + +def test_widget_exchange_never_sends_keys_and_checks_the_current_slot(): + """The real receiving method keeps only keys still present on the last pushed slot.""" + secret = "RGSYNTH_widget_20261103" + s = rustgis.Session() + s.add_geojson(POINT, name="Sites") + project = json.loads(s.project_json()) + project["maps"][0]["layers"][0]["dataset"]["source"] = f"https://h.invalid/a?token={secret}" + s.run("project.openText", text=json.dumps(project)) + saved = s.project_json() + assert secret not in saved + other = rustgis.Session() + other.run("project.openText", text=saved) + assert other._engine._peer_text()[1] == 1 + m = rustgis.Map(session=s) + assert secret not in m._project + assert any("without their keys" in notice for notice, _ in m.notices()) + incoming = json.loads(m._project) + incoming["maps"][0]["layers"][0]["name"] = "Styled by app" + from_app(m, json.dumps(incoming)) + assert m.session._engine._peer_text()[1] == 1 + assert not any("lost their keys" in notice for notice, _ in m.notices()) + # Direct session changes do not call the widget's push. Its stale record cannot resurrect + # the removed source when the app sends the old slot again. + m.session.run("layer.remove", layer="Styled by app") + incoming["maps"][0]["layers"][0]["name"] = "Stale app slot" + from_app(m, json.dumps(incoming)) + assert m.session._engine._peer_text()[1] == 0 + + +def test_widget_repeated_table_id_cannot_gain_a_retained_key(): + """The receiving census sees duplicate table ids before project normalization renumbers them.""" + s = rustgis.Session() + project = json.loads(s.project_json()) + project["tables"] = [{"id":7, "name":"Private table", "geometry_type":"None", "fields":[], "features":[], "source":"https://h.invalid/table?token=RGSYNTH_table_20261103"}] + s.run("project.openText", text=json.dumps(project)) + m = rustgis.Map(session=s) + incoming = json.loads(m._project) + incoming["tables"].append(dict(incoming["tables"][0])) + from_app(m, json.dumps(incoming)) + assert s._engine._peer_text()[1] == 0 + assert any("lost their keys" in notice for notice, _ in m.notices()) diff --git a/apps/rustgis-web/src/embed.rs b/apps/rustgis-web/src/embed.rs index dbe30f7c4..4ab16a601 100644 --- a/apps/rustgis-web/src/embed.rs +++ b/apps/rustgis-web/src/embed.rs @@ -17,6 +17,11 @@ //! | `{"rustgis": "project", "text": }` | the project changed here (an edit, a pan) | //! | `{"rustgis": "result", "id", "ok": true, "value"}` or `{…, "ok": false, "error"}` | a `run`'s outcome | //! +//! Projects exchanged here contain public configuration without credentials. Incoming projects +//! use peer/import semantics: no identical-save restore, item trust or earlier approvals. Parent +//! origin checks identify the peer; they grant no user authority. The existing `ui.*` command +//! policy is separate from this project-exchange contract. +//! //! Only messages from the parent window are taken, and the app answers only the origin that sent //! the first of them. Any page may frame the app, so `run` takes only the interface's commands //! (`ui.*`: panes, the language, the theme); everything else changes the project, which the page diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index dd3d53069..097027844 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -395,7 +395,7 @@ impl Embed { let queued = std::mem::take(&mut *self.queue.borrow_mut()); for value in queued { match parse(&value) { - Some(Msg::Project(text)) => match app.run("project.openText", serde_json::json!({"text": text})) { + Some(Msg::Project(text)) => match app.open_peer_text(&text) { Ok(_) => self.changes.seen(key(&app.session)), Err(e) => app.session.notify(e, true), }, @@ -411,8 +411,13 @@ impl Embed { } } match self.changes.due(key(&app.session), web_time::Instant::now()) { - (true, _) => match serde_json::to_string(&app.session.project) { - Ok(text) => self.answer(&crate::embed::project(&text)), + (true, _) => match app.session.peer_text(rustgis_engine::Peer::Embedder) { + Ok((text, report)) => { + self.answer(&crate::embed::project(&text)); + for notice in report.notices() { + app.session.notify(notice, false); + } + } Err(e) => log::error!("couldn't send the project: {e}"), }, (false, Some(wait)) => ctx.request_repaint_after(wait), diff --git a/crates/assistant/src/agent.rs b/crates/assistant/src/agent.rs index 444a6500d..2a337d25c 100644 --- a/crates/assistant/src/agent.rs +++ b/crates/assistant/src/agent.rs @@ -942,6 +942,21 @@ impl Agent { self.drain(s) } + /// Peer replacement invalidates pending work and remembered approvals without deleting the + /// conversation. An outstanding provider reply belongs to the previous project generation. + pub fn project_replaced(&mut self) { + self.stop(); + self.state = State::Idle; + self.always.clear(); + self.last_context.clear(); + self.preview = None; + self.retrying_summary = None; + self.child = None; + self.delegations.clear(); + self.reports.clear(); + self.generation = self.generation.wrapping_add(1); + } + /// Stops the run: a request out is forgotten, calls not yet run are skipped. pub fn stop(&mut self) { if !self.busy() { diff --git a/crates/assistant/src/audit.rs b/crates/assistant/src/audit.rs index 2c2523fc7..b047aafdb 100644 --- a/crates/assistant/src/audit.rs +++ b/crates/assistant/src/audit.rs @@ -89,7 +89,8 @@ pub fn entries(source: &str, request: &str, reply: Option<&str>, took: Duration, .filter(|m| m.get("method").and_then(Value::as_str) == Some("tools/call") && m.get("id").is_some()) .map(|m| { let params = m.get("params").cloned().unwrap_or(Value::Null); - let args = redact(params.get("arguments").cloned().unwrap_or(Value::Null), 0); + let args = + normalized_args(params.get("name").and_then(Value::as_str).unwrap_or(""), params.get("arguments").unwrap_or(&Value::Null)); let r = replies.iter().find(|r| r.get("id") == m.get("id")); let failed = r.is_none_or(|r| r.get("error").is_some() || r.pointer("/result/isError") == Some(&Value::Bool(true))); let error = failed.then(|| { @@ -138,56 +139,24 @@ pub fn entries(source: &str, request: &str, reply: Option<&str>, took: Duration, .collect() } -/// The words that name a secret in a key's name, whole words and their plurals. -const SECRET_WORDS: &[&str] = &[ - "passphrase", - "jwt", - "privatekey", - "password", - "passwd", - "pwd", - "secret", - "token", - "apikey", - "auth", - "authorization", - "bearer", - "credential", - "signature", - "cookie", - "sas", - "dsn", -]; -/// Words before "key" that make it a secret key (`api_key`, `accessKey`), not a sort or join key. -const KEY_KINDS: &[&str] = &["api", "access", "private", "secret", "account", "sas", "subscription", "client"]; - -/// Whether a key's name names a secret: its words (split at `_`, `-`, `.` and camelCase), one of -/// [`SECRET_WORDS`] or its plural, or "key" after one of [`KEY_KINDS`] (or alone). -fn secret_key(name: &str) -> bool { - let mut words: Vec = Vec::new(); - let mut cur = String::new(); - let mut prev_lower = false; - for c in name.chars() { - if c == '_' || c == '-' || c == '.' || c == ' ' { - words.push(std::mem::take(&mut cur)); - prev_lower = false; - continue; - } - if c.is_ascii_uppercase() && prev_lower { - words.push(std::mem::take(&mut cur)); +/// Arguments as the executor reads them. Only this output copy is redacted; it is never +/// supplied to the executor, the permission planner or approval equality. +fn normalized_args(tool: &str, arguments: &Value) -> Value { + match tool { + "run_command" | "run_tool" => { + let params = match rustgis_engine::params_envelope(arguments.get("params")) { + Ok(params) => rustgis_engine::log_params(¶ms).into_log_value(), + Err(_) => json!("[parameters not JSON; withheld]"), + }; + let key = if tool == "run_command" { "command" } else { "tool" }; + let mut out = serde_json::Map::new(); + out.insert(key.into(), redact(arguments.get(key).cloned().unwrap_or(Value::Null), 0)); + out.insert("params".into(), params); + Value::Object(out) } - prev_lower = c.is_ascii_lowercase() || c.is_ascii_digit(); - cur.push(c.to_ascii_lowercase()); + "run" => json!({"script": redact_script(arguments.get("script").and_then(Value::as_str).unwrap_or(""))}), + _ => redact(arguments.clone(), 0), } - words.push(cur); - let words: Vec<&str> = words.iter().map(String::as_str).filter(|w| !w.is_empty()).collect(); - let is = |w: &str, s: &str| w == s || w.strip_suffix('s') == Some(s); - words.iter().enumerate().any(|(i, w)| { - SECRET_WORDS.iter().any(|s| is(w, s)) - // Run together without a separator (`accesstoken`, `clientsecret`, `dbpassword`). - || ["token", "secret", "password", "passphrase"].iter().any(|s| w.len() > s.len() && (w.ends_with(s) || w.ends_with(&format!("{s}s")))) - || (is(w, "key") && (words.len() == 1 || i.checked_sub(1).and_then(|j| words.get(j)).is_some_and(|p| KEY_KINDS.contains(p)))) - }) } /// A string argument without its secrets: an address (one word with a scheme, or a database @@ -215,7 +184,15 @@ fn redact_string(s: &str) -> String { // Text with an address in it (a query, a description): each address word that carries // credentials (a user and password, or a key or token in its query) loses them; the rest, // and the spacing, stay as they were. - let carries = |w: &str| address(w) && (w.contains('@') || mentions_secret(w) || w.to_ascii_lowercase().contains("key=")); + let carries = |w: &str| { + let q = w.split_once('?').map_or("", |(_, q)| q.split('#').next().unwrap_or(q)); + let names = rustgis_engine::data::address::QueryNames::of(q); + let secret_pair = q.split(['&', ';']).any(|pair| { + let name = pair.split_once('=').map_or(pair, |(k, _)| k); + rustgis_engine::data::address::name_class(name, &names) != rustgis_engine::data::address::NameClass::Plain + }); + address(w) && (w.contains('@') || mentions_secret(w) || secret_pair) + }; if t.split_whitespace().any(carries) { let mut out = String::with_capacity(s.len()); let mut word = String::new(); @@ -239,7 +216,7 @@ fn redact_string(s: &str) -> String { s.to_string() } -/// Whether free text names a secret. Narrower than [`SECRET_WORDS`]: a plain "key" in an error +/// Whether free text names a secret. Narrower than the structured-name list: a plain "key" in an error /// ("primary key") isn't one. fn mentions_secret(text: &str) -> bool { let lower = text.to_ascii_lowercase(); @@ -254,18 +231,16 @@ fn mentions_secret(text: &str) -> bool { fn redact_script(script: &str) -> String { script .lines() - .map(|line| { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - return line.to_string(); + .take(rustgis_engine::cmd::workflows::MAX_SCRIPT_LINES) + .map(|line| match rustgis_engine::script_line(line) { + None => line.trim().to_string(), + Some(Ok((id, params))) => { + let id = redact_string(&id); + if params.is_null() { id } else { format!("{id} {}", rustgis_engine::log_params(¶ms).into_log_value()) } } - let (id, rest) = line.split_once(char::is_whitespace).unwrap_or((line, "")); - if rest.trim().is_empty() { - return id.to_string(); - } - match serde_json::from_str::(rest.trim()) { - Ok(params) => format!("{id} {}", redact(params, 0)), - Err(_) => format!("{id} [parameters not JSON; withheld]"), + Some(Err(_)) => { + let id = line.trim().split_once(char::is_whitespace).map_or(line.trim(), |(id, _)| id).trim_end_matches('='); + format!("{} [parameters not JSON; withheld]", redact_string(id)) } }) .collect::>() @@ -283,7 +258,7 @@ fn redact(v: Value, depth: usize) -> Value { Value::Array(a) if depth < 16 => Value::Array(a.into_iter().map(|x| redact(x, depth + 1)).collect()), Value::Object(o) if depth < 16 => Value::Object( o.into_iter() - .map(|(k, x)| match (secret_key(&k), k.as_str(), x) { + .map(|(k, x)| match (rustgis_engine::data::address::log_sensitive(&k), k.as_str(), x) { (true, _, _) => (k, json!("[redacted]")), (false, "script", Value::String(s)) => (k, json!(redact_script(&s))), (false, _, x) => (k, redact(x, depth + 1)), @@ -329,6 +304,37 @@ fn prune(dir: &Path, secs: i64) { mod tests { use super::*; + #[test] + fn audit_logs_the_executed_envelope_and_shared_script_grammar() { + let params = + json!({"url": "https://h.invalid/a?%74oken=RGSYNTH_audit_20261103", "pass%77ord": "RGSYNTH_option_20261103", "layer": "roads"}); + for tool in ["run_command", "run_tool"] { + let object = json!({"command": "layer.addUrl", "tool": "buffer", "params": params}); + let mut string = object.clone(); + string["params"] = json!(params.to_string()); + let logged = normalized_args(tool, &object); + assert_eq!(logged, normalized_args(tool, &string)); + assert_eq!(logged["params"], rustgis_engine::log_params(¶ms).into_log_value()); + assert!(!logged.to_string().contains("RGSYNTH_")); + for bad in [json!("{broken RGSYNTH_bad_20261103"), json!("[1]"), json!(42)] { + let mut bad_args = object.clone(); + bad_args["params"] = bad; + assert_eq!(normalized_args(tool, &bad_args)["params"], "[parameters not JSON; withheld]"); + } + } + assert!(!redact_string("failed https://h.invalid/a?%74oken=RGSYNTH_error_20261103").contains("RGSYNTH_")); + let script = "map.zoomIn= {}\nlayer.addUrl {broken RGSYNTH_bad_20261103\nmap.zoomOut {}"; + let logged = redact_script(script); + assert!(logged.starts_with("map.zoomIn {}\nlayer.addUrl [parameters not JSON; withheld]\nmap.zoomOut {}")); + assert!(!logged.contains("RGSYNTH_")); + let document = + json!({"type": "FeatureCollection", "features": [], "padding": "x".repeat(250), "token": "RGSYNTH_data_20261103"}).to_string(); + let arguments = json!({"command": "layer.addDataText", "params": {"text": document}}); + let logged = normalized_args("run_command", &arguments); + assert!(logged["params"]["text"].as_str().is_some_and(|s| s.starts_with("(a document of "))); + assert_eq!(arguments["params"]["text"], document, "the executed data document stays intact"); + } + /// A line for each tool call: what ran, whether it worked, how long it took; secrets in its /// arguments are not written. #[test] @@ -389,10 +395,10 @@ mod tests { "passphrase", "jwt", ] { - assert!(secret_key(k), "{k}"); + assert!(rustgis_engine::data::address::log_sensitive(k), "{k}"); } for k in ["primary_key", "sort_key", "keys_column", "texas", "name"] { - assert!(!secret_key(k), "{k}"); + assert!(!rustgis_engine::data::address::log_sensitive(k), "{k}"); } // Free text keeps its shape; a keyword connection string loses its password. assert_eq!(redact_string("SELECT * FROM t WHERE url = 'http://x?a=1#b'"), "SELECT * FROM t WHERE url = 'http://x?a=1#b'"); diff --git a/crates/assistant/src/tests.rs b/crates/assistant/src/tests.rs index 17883012e..7a4c514cf 100644 --- a/crates/assistant/src/tests.rs +++ b/crates/assistant/src/tests.rs @@ -4300,3 +4300,21 @@ fn geolibre_download_callbacks_require_import_approval() { assert!(tools::gate(&s, &call).is_some(), "{key}"); } } + +/// A peer project invalidates queued work and replies while keeping the readable conversation. +#[test] +fn peer_replacement_cancels_approvals_and_stale_replies() { + let mut s = sample(); + let mut agent = Agent::new(cfg("openai"), &s); + assert!(matches!(agent.ask("export the map", &s), Step::Send(_))); + let pending = + call("peer-old", "run_command", json!({"command":"share.exportMap", "params":{"path":"RGSYNTH_never_written_20261103.png"}})); + assert_eq!(agent.receive(agent.generation(), ok(pending), &mut s), Step::Confirm); + let generation = agent.generation(); + let transcript = agent.transcript.len(); + agent.project_replaced(); + assert!(agent.generation() != generation && !agent.busy() && agent.pending(&s).is_none()); + assert!(agent.transcript.len() >= transcript); + let _ = agent.receive(generation, ok(say("stale")), &mut s); + assert!(!agent.busy() && agent.pending(&s).is_none()); +} diff --git a/crates/assistant/src/tools.rs b/crates/assistant/src/tools.rs index 97e93b57f..c25b42dff 100644 --- a/crates/assistant/src/tools.rs +++ b/crates/assistant/src/tools.rs @@ -387,20 +387,10 @@ fn str_arg(a: &Value, k: &str) -> Option { /// The `params` a command or tool runs with: an object, or JSON text of one (some models send /// that). Anything else is an error: running with defaults instead could act on the wrong layer. fn params_arg(a: &Value) -> Result { - match a.get("params") { - None | Some(Value::Null) => Ok(json!({})), - Some(v @ Value::Object(_)) => Ok(v.clone()), - Some(Value::String(t)) if t.trim().is_empty() => Ok(json!({})), - Some(Value::String(t)) => match serde_json::from_str::(t) { - Ok(v @ Value::Object(_)) => Ok(v), - Ok(_) => Err("params must be a JSON object".into()), - Err(e) => Err(format!("params is not valid JSON: {e}")), - }, - Some(_) => Err("params must be a JSON object".into()), - } + rustgis_engine::params_envelope(a.get("params")) } -/// An object parameter, which some models send as JSON text (leniently, for display and approval). +/// An object parameter, which some models send as JSON text (leniently, for display only). fn obj_arg(a: &Value, k: &str) -> Value { match a.get(k) { Some(Value::String(t)) if !t.trim().is_empty() => serde_json::from_str(t).unwrap_or(Value::Null), diff --git a/crates/data/src/address.rs b/crates/data/src/address.rs new file mode 100644 index 000000000..17a82f52d --- /dev/null +++ b/crates/data/src/address.rs @@ -0,0 +1,676 @@ +//! Credential names and public address text, shared by sources, diagnostics and writers. +//! +//! A plain name is not a promise about its value. In particular it does not authorize sending +//! that value to an address advertised by a service. These helpers never inspect dataset text. + +use std::borrow::Cow; + +/// Maximum decoding work for one name; anything still encoded is withheld. +pub const MAX_DECODE_PASSES: usize = 4; +pub const MAX_NAME_BYTES: usize = 16 * 1024; + +/// Classification after bounded decoding, without decoding a pair's value. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NameClass { + Credential, + /// Not recognized; this alone never authorizes forwarding. + Plain, + /// Too long, or still decodable after the budget. Never forwarded, even under a scope. + Unresolved, +} + +/// Parses only a complete hexadecimal escape supplied by the byte scanner. +fn hex(bytes: &[u8]) -> Option { + bytes.iter().try_fold(0, |n, b| Some(n * 16 + char::from(*b).to_digit(16)?)) +} + +/// One byte-preserving pass. Lossy UTF-8 is applied only after all passes, so percent-encoded +/// bytes and literal Unicode take the same route. Invalid escapes remain literal. +fn decode_pass(bytes: &[u8]) -> Vec { + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + if b == b'%' + && let Some(n) = bytes.get(i + 1..i + 3).and_then(hex) + { + out.push(n as u8); + i += 3; + } else if b == b'\\' + && bytes.get(i + 1) == Some(&b'u') + && let Some(c) = bytes.get(i + 2..i + 6).and_then(hex).and_then(char::from_u32) + { + out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()); + i += 6; + } else { + out.push(if b == b'+' { b' ' } else { b }); + i += 1; + } + } + out +} + +/// Repeated decoding stops at a fixed point; a fifth changing pass is unresolved. +fn decoded_name(raw: &str) -> Option> { + if raw.len() > MAX_NAME_BYTES { + return None; + } + let mut name = raw.as_bytes().to_vec(); + for _ in 0..MAX_DECODE_PASSES { + let next = decode_pass(&name); + if next == name { + return Some(name); + } + name = next; + } + (decode_pass(&name) == name).then_some(name) +} + +/// Canonical comparison spelling; invalid UTF-8 contributes no ASCII name characters. +fn letters(bytes: &[u8]) -> String { + String::from_utf8_lossy(bytes).chars().filter(char::is_ascii_alphanumeric).map(|c| c.to_ascii_lowercase()).collect() +} + +/// ASCII letters and digits after bounded decoding, lower-cased; `None` means Unresolved. +pub fn normalized_name(raw: &str) -> Option { + decoded_name(raw).map(|s| letters(&s)) +} + +/// Context is collected from decoded query names, including legacy semicolon-separated pairs. +#[derive(Default)] +pub struct QueryNames { + sig: bool, + sv: bool, + state: bool, +} + +impl QueryNames { + /// Collects context flags without decoding values or retaining query payloads. + pub fn of(query: &str) -> Self { + let mut names = Self::default(); + for pair in query.split(['&', ';']) { + if let Some(decoded) = decoded_name(pair_name(pair)) { + let k = letters(decoded.split(|b| *b == b'=').next().unwrap_or(&decoded)); + names.sig |= k == "sig"; + names.sv |= k == "sv"; + names.state |= k == "state"; + } + } + names + } +} + +const SAS_KEYS: &[&str] = + &["sv", "se", "sp", "sr", "st", "spr", "sip", "si", "ss", "srt", "skoid", "sktid", "skt", "ske", "sks", "skv", "sdd"]; + +/// Exact and containment rules over canonical names, plus query-wide context. +fn credential(k: &str, query: &QueryNames) -> bool { + matches!(k, "key" | "pwd" | "sig" | "signature" | "sas" | "authorization" | "proxyauthorization" | "cookie" | "setcookie" | "bearer") + // These public refusal and tracing headers share the signing family's prefix. + || (k.starts_with("xamz") && !matches!(k, "xamznrequestid" | "xamznwafaction" | "xamzcfid")) + || k.starts_with("xgoog") + || [ + "password", + "passwd", + "secret", + "token", + "credential", + "apikey", + "accesskey", + "privatekey", + "accountkey", + "sharedaccess", + "connectionstring", + ] + .iter() + .any(|w| k.contains(w)) + || (query.sig && query.sv && SAS_KEYS.contains(&k)) + || (k == "code" && query.state) +} + +/// Classifies a raw name. An encoded equals sign is checked both as a name boundary and as +/// part of the whole name; values after a literal equals sign are never decoded here. +pub fn name_class(raw: &str, query: &QueryNames) -> NameClass { + let Some(decoded) = decoded_name(raw) else { return NameClass::Unresolved }; + let before = decoded.split(|b| *b == b'=').next().unwrap_or(&decoded); + if credential(&letters(&decoded), query) || credential(&letters(before), query) { NameClass::Credential } else { NameClass::Plain } +} + +/// True for recognized credentials and unresolved names; no query context is assumed. +pub fn secret_key(raw: &str) -> bool { + name_class(raw, &QueryNames::default()) != NameClass::Plain +} + +/// Names logs also omit, beyond credential-bearing address options. Logs are output only. +pub fn log_sensitive(raw: &str) -> bool { + if secret_key(raw) { + return true; + } + let Some(decoded) = decoded_name(raw) else { return true }; + let name = String::from_utf8_lossy(&decoded); + let k = letters(&decoded); + if k.contains("body") + || matches!(k.as_str(), "basic" | "authentication" | "wwwauthenticate" | "proxyauthenticate" | "userid" | "username") + { + return true; + } + // The audit's existing word policy, including plurals and camelCase, is shared here with + // diagnostics. It deliberately omits more than replayable state does. + let mut words = Vec::new(); + let mut word = String::new(); + let mut prev_lower = false; + for c in name.chars() { + if (!c.is_ascii_alphanumeric() || (c.is_ascii_uppercase() && prev_lower)) && !word.is_empty() { + words.push(std::mem::take(&mut word)); + } + prev_lower = c.is_ascii_lowercase() || c.is_ascii_digit(); + if c.is_ascii_alphanumeric() { + word.push(c.to_ascii_lowercase()); + } + } + if !word.is_empty() { + words.push(word); + } + words.iter().enumerate().any(|(i, word)| { + let w = word.strip_suffix('s').unwrap_or(word); + matches!( + w, + "passphrase" + | "jwt" + | "privatekey" + | "password" + | "passwd" + | "pwd" + | "secret" + | "token" + | "apikey" + | "auth" + | "authorization" + | "bearer" + | "credential" + | "signature" + | "cookie" + | "sas" + | "dsn" + ) || ["token", "secret", "password", "passphrase"].iter().any(|s| w.ends_with(s)) + || (w == "key" + && (words.len() == 1 + || i.checked_sub(1).and_then(|j| words.get(j)).is_some_and(|p| { + ["api", "access", "private", "secret", "account", "sas", "subscription", "client"].contains(&p.as_str()) + }))) + }) +} + +/// Forwarding options compare their complete decoded spelling, including punctuation. The +/// broader credential normalization must not make an unknown name into an allowlisted one. +fn option_name(raw: &str) -> Option { + decoded_name(raw).map(|s| String::from_utf8_lossy(&s).to_ascii_lowercase()) +} + +/// Only a literal equals sign separates a raw name from its value. +fn pair_name(pair: &str) -> &str { + pair.split_once('=').map_or(pair, |(k, _)| k) +} + +/// A legacy semicolon credential taints the whole ampersand pair. +fn pair_class(pair: &str, names: &QueryNames) -> NameClass { + let mut class = NameClass::Plain; + for sub in pair.split(';') { + match name_class(pair_name(sub), names) { + NameClass::Unresolved => return NameClass::Unresolved, + NameClass::Credential => class = NameClass::Credential, + NameClass::Plain => {} + } + } + class +} + +/// Query text ends at the fragment, which is never interpreted as another query. +fn query(address: &str) -> &str { + address.split('#').next().and_then(|s| s.split_once('?')).map_or("", |(_, q)| q) +} + +/// Removes credential and unresolved pairs; retained pairs keep their original bytes. The +/// caller handles user-info and fragments. A legacy `;token=…` removes its entire `&` pair. +pub fn redact_query(address: &mut String) { + let Some(at) = address.find('?').filter(|at| address.find('#').is_none_or(|end| *at < end)) else { return }; + let q = query(address); + let names = QueryNames::of(q); + if q.split('&').all(|p| !p.is_empty() && pair_class(p, &names) == NameClass::Plain) { + return; + } + let kept = q.split('&').filter(|p| !p.is_empty() && pair_class(p, &names) == NameClass::Plain).collect::>().join("&"); + let fragment = address.find('#').and_then(|i| address.get(i..)).unwrap_or("").to_string(); + address.truncate(at); + if !kept.is_empty() { + address.push('?'); + address.push_str(&kept); + } + address.push_str(&fragment); +} + +/// Public form of one web or S3 address. Local paths and other unrecognized text are untouched. +#[derive(Debug, PartialEq, Eq)] +pub enum Public { + Unchanged, + Changed { + address: String, + removed: u16, + }, + /// A recognized address whose authority cannot safely be delimited; retain only its prefix. + Withheld, +} + +/// Masks supported tile placeholders in a validation copy of the authority. Equal byte lengths +/// preserve user-info offsets; the public address always retains the original template spelling. +/// Switch choices may name host labels, but cannot introduce URL delimiters or credentials. +fn template_authority(rest: &str) -> Cow<'_, str> { + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let Some(authority) = rest.get(..end) else { return Cow::Borrowed(rest) }; + let mut masked = None; + let mut copied = 0; + let mut scan = 0; + while let Some(open) = authority.get(scan..).and_then(|s| s.find('{')).map(|at| scan + at) { + let Some(close) = authority.get(open + 1..).and_then(|s| s.find('}')).map(|at| open + 1 + at) else { break }; + let token = authority.get(open + 1..close).unwrap_or(""); + let known = matches!( + token, + "z" | "zoom" | "x" | "y" | "-y" | "q" | "quadkey" | "s" | "bbox-epsg-3857" | "width" | "height" | "r" | "ratio" + ) || token.strip_prefix("switch:").is_some_and(|choices| { + choices.split(',').all(|choice| { + let choice = choice.trim(); + !choice.is_empty() && choice.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_')) + }) + }); + if known { + let out = masked.get_or_insert_with(|| String::with_capacity(rest.len())); + out.push_str(authority.get(copied..open).unwrap_or("")); + out.extend(std::iter::repeat_n('x', close + 1 - open)); + copied = close + 1; + } + scan = close + 1; + } + match masked { + Some(mut out) => { + out.push_str(rest.get(copied..).unwrap_or("")); + Cow::Owned(out) + } + None => Cow::Borrowed(rest), + } +} + +/// Removes user-info, fragments and secret query pairs, or withholds an unreadable authority. +pub fn public_web_address(address: &str) -> Public { + let text = address.trim(); + let Some((protocol, _)) = text.split_once(':') else { return Public::Unchanged }; + if !["http", "https", "s3"].iter().any(|p| protocol.eq_ignore_ascii_case(p)) { + return Public::Unchanged; + } + let Some((scheme, rest)) = text.split_once("://") else { return Public::Withheld }; + let at = crate::source::pgconn::userinfo_end(&template_authority(rest)); + let rest = at.and_then(|i| rest.get(i + 1..)).unwrap_or(rest); + let no_fragment = rest.split('#').next().unwrap_or(rest); + let candidate = format!("{scheme}://{no_fragment}"); + if no_fragment.split(['/', '?']).next().is_none_or(str::is_empty) + // Backslashes can change authority/path parsing, but query backslashes are data: + // escaped credential names still need the bounded query decoder below. + || candidate.split('?').next().is_some_and(|head| head.contains('\\')) + || candidate.chars().any(char::is_control) + || url::Url::parse(&format!("{scheme}://{}", template_authority(no_fragment))) + .ok() + .is_none_or(|u| u.host_str().is_none() || !u.username().is_empty() || u.password().is_some()) + { + return Public::Withheld; + } + let names = QueryNames::of(query(&candidate)); + let removed = query(&candidate).split('&').filter(|p| !p.is_empty() && pair_class(p, &names) != NameClass::Plain).count(); + let removed = + u16::try_from(removed).unwrap_or(u16::MAX).saturating_add(u16::from(at.is_some())).saturating_add(u16::from(rest.contains('#'))); + let mut public = candidate; + redact_query(&mut public); + if public == address { Public::Unchanged } else { Public::Changed { address: public, removed } } +} + +/// Cleans address words in generated text and counts changed words. All spacing and all other +/// text stay byte-for-byte. Never apply this to user-authored text or dataset values. +pub fn scrub_addresses(text: &str) -> (Cow<'_, str>, u32) { + let mut changed = 0u32; + let mut out = String::new(); + let mut start = 0; + for word in text.split_inclusive(char::is_whitespace) { + let unspaced = word.trim_end_matches(char::is_whitespace); + // Recognize supported prefixes before parsing: malformed delimiters must not turn a + // recognized address into retained text. The preceding scheme characters establish the + // same word boundary for a bare address, a quoted one and an address inside a wrapper. + let Some((begin, scheme_at)) = unspaced.match_indices(':').find_map(|(at, _)| { + let begin = unspaced + .get(..at)? + .char_indices() + .rev() + .find(|(_, c)| !c.is_ascii_alphanumeric() && !"+.-".contains(*c)) + .map_or(0, |(i, c)| i + c.len_utf8()); + let scheme = unspaced.get(begin..at)?; + ["http", "https", "s3"].iter().any(|s| scheme.eq_ignore_ascii_case(s)).then_some((begin, at)) + }) else { + if changed > 0 { + out.push_str(word); + } + start += word.len(); + continue; + }; + let end = unspaced.trim_end_matches([':', ',', ';', '.', ')', ']', '>', '"', '\'', '»', '›', '”', '’']).len().max(scheme_at + 1); + let address = word.get(begin..end).unwrap_or(""); + let public = public_web_address(address); + if !matches!(public, Public::Unchanged) { + if changed == 0 { + out.push_str(text.get(..start).unwrap_or("")); + } + changed = changed.saturating_add(1); + } + if changed > 0 { + out.push_str(word.get(..begin).unwrap_or("")); + match public { + Public::Unchanged => out.push_str(address), + Public::Changed { address, .. } => out.push_str(&address), + Public::Withheld => { + out.push_str(address.split(':').next().unwrap_or("https")); + out.push(':'); + } + } + out.push_str(word.get(end..).unwrap_or("")); + } + start += word.len(); + } + (if changed == 0 { Cow::Borrowed(text) } else { Cow::Owned(out) }, changed) +} + +/// Mechanical path containment for the exact address fetched. Containment alone grants no +/// authority: advertised destinations still use F3 unless a deployment was explicitly reviewed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Scope { + scheme: String, + host: String, + port: u16, + segments: Vec, +} + +impl Scope { + /// Reads authority with the fetch parser, and path segments from the original bytes. + pub fn of(address: &str) -> Option { + if address.trim() != address || address.chars().any(char::is_control) || address.contains('\\') { + return None; + } + let parsed = url::Url::parse(address).ok()?; + if !matches!(parsed.scheme(), "http" | "https") || !parsed.username().is_empty() || parsed.password().is_some() { + return None; + } + let (_, rest) = address.split_once("://")?; + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + if end == 0 || rest.get(..end)?.contains('@') { + return None; + } + let path = rest.get(end..)?.split(['?', '#']).next()?; + let path = path.strip_prefix('/').unwrap_or(path); + let path = path.strip_suffix('/').unwrap_or(path); + let mut segments = Vec::new(); + if !path.is_empty() { + for segment in path.split('/') { + let lower = segment.to_ascii_lowercase(); + if segment.is_empty() || matches!(segment, "." | "..") || ["%2e", "%2f", "%5c", "%25"].iter().any(|s| lower.contains(s)) { + return None; + } + segments.push(segment.to_string()); + } + } else if rest.get(end..)?.starts_with("//") { + return None; + } + Some(Self { + scheme: parsed.scheme().to_string(), + host: parsed.host_str()?.trim_end_matches('.').to_ascii_lowercase(), + port: parsed.port_or_known_default()?, + segments, + }) + } + + /// Whole raw segments must match; ambiguous paths and user-info are outside every scope. + pub fn covers_below(&self, to: &str) -> bool { + Self::of(to).is_some_and(|other| { + self.scheme == other.scheme && self.host == other.host && self.port == other.port && other.segments.starts_with(&self.segments) + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Relation { + Constructed, + Advertised, +} + +pub const SAFE_OPTIONS: &[&str] = &["collections", "datetime", "subset", "subset-crs", "crs", "bgcolor", "transparent"]; + +/// Only a reviewed deployment may authorize full-query inheritance for advertised links. Root +/// entries require separate whole-origin agreement, including every tenant on that origin. +pub struct ReviewedScope { + pub directory: &'static str, + pub whole_origin: bool, +} +pub const F2_SCOPES: &[ReviewedScope] = &[]; + +#[derive(Debug, PartialEq, Eq)] +pub struct Inherited { + pub url: String, + pub withheld: bool, +} + +/// Adds missing pairs, preserving their bytes. Unresolved pairs never follow; F3 sends only +/// SAFE_OPTIONS to advertised addresses. Only checked constructed sub-addresses get other pairs. +pub fn inherit_query(from: &str, to: &str, relation: Relation, skip: &[&str]) -> Inherited { + inherit_reviewed(from, to, relation, skip, F2_SCOPES) +} + +/// Applies the fixed policy with an explicit reviewed list; production passes the empty list. +fn inherit_reviewed(from: &str, to: &str, relation: Relation, skip: &[&str], reviewed: &[ReviewedScope]) -> Inherited { + let full = match relation { + Relation::Constructed => Scope::of(from).is_some_and(|s| s.covers_below(to)), + Relation::Advertised => reviewed.iter().any(|r| { + Scope::of(r.directory).is_some_and(|s| (!s.segments.is_empty() || r.whole_origin) && s.covers_below(from) && s.covers_below(to)) + }), + }; + let q = query(from); + let names = QueryNames::of(q); + let mut out = Inherited { url: to.to_string(), withheld: false }; + let mut present: Vec = query(to).split(['&', ';']).filter_map(|p| option_name(pair_name(p))).collect(); + for pair in q.split('&').filter(|p| !p.is_empty()) { + let class = pair_class(pair, &names); + if class == NameClass::Unresolved { + out.withheld = true; + continue; + } + let keys: Vec = pair.split(';').filter_map(|p| option_name(pair_name(p))).collect(); + let skipped = |k: &String| skip.iter().any(|s| option_name(s).as_ref() == Some(k)); + if keys.iter().any(skipped) { + out.withheld |= keys.iter().any(|k| !skipped(k)); + continue; + } + if keys.iter().any(|k| present.contains(k)) { + // Never overwrite a destination's pair, including a legacy sub-pair. If only some + // names overlap, the rest of this indivisible raw pair were withheld too. + out.withheld |= keys.iter().any(|k| !present.contains(k)); + continue; + } + let safe = class == NameClass::Plain && keys.iter().all(|k| SAFE_OPTIONS.contains(&k.as_str())); + if pair.contains(['{', '}']) || !(full || safe) { + out.withheld = true; + continue; + } + let fragment = out.url.find('#').unwrap_or(out.url.len()); + let separator = if out.url.get(..fragment).is_some_and(|u| u.contains('?')) { '&' } else { '?' }; + out.url.insert(fragment, separator); + out.url.insert_str(fragment + 1, pair); + present.extend(keys); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_names_remove_encoded_and_unresolved_pairs_without_touching_values() { + let over = "a".repeat(MAX_NAME_BYTES + 1); + for key in [ + "%74oken", + "%2574oken", + "to%6Ben", + "pass%77ord", + "+token", + "token%3DM", + "%FFtoken", + "ok=1;token", + "X-Amz-%53ignature", + "X-Amz-%53ecurity-Token", + "X-Amz-Credential", + "X-Amzn-RequestId-Token", + "X-Amz-Cf-Id-Secret", + "sig=M&%73v", + "\\u0074oken", + "accessKeyId", + "%2525252574oken", + over.as_str(), + ] { + let address = format!("https://h.invalid/a?{key}=RGSYNTH_name_20261103&layer=a%20b&f=json&%zz=1"); + let Public::Changed { address: public, removed, .. } = public_web_address(&address) else { panic!("{key}") }; + assert!(!public.contains("RGSYNTH_name_20261103"), "{key}: {public}"); + assert!(public.ends_with("layer=a%20b&f=json&%zz=1"), "{public}"); + assert!(removed > 0); + } + for query in ["layer=a%20b&f=json&%zz=1", "layer=token%3DM", "code=EPSG:4326", "sv=2026&layer=1", r"layer=a\b&f=json"] { + let address = format!("https://h.invalid/a?{query}"); + assert_eq!(public_web_address(&address), Public::Unchanged); + } + for name in ["X-Amzn-RequestId", "X-Amzn-Waf-Action", "X-Amz-Cf-Id", "X-%2541mzn-RequestId"] { + assert_eq!(name_class(name, &QueryNames::default()), NameClass::Plain, "{name}"); + assert!(!log_sensitive(name), "{name}"); + } + for address in [r"https://h.invalid\a?token=RGSYNTH_boundary", r"https://h.invalid/a\b?token=RGSYNTH_boundary"] { + assert_eq!(public_web_address(address), Public::Withheld); + } + for template in ["https://{switch:a,b}.host/{z}/{x}/{y}@2x.png?map=m", "https://{s}.host/{zoom}/{x}/{y}@2x.png"] { + assert_eq!(public_web_address(template), Public::Unchanged); + } + assert_eq!( + public_web_address( + "https://u:RGSYNTH_template_20261104@{switch:a,b}.host/{z}/{x}/{y}@2x.png?token=RGSYNTH_query_20261104&map=m#fragment" + ), + Public::Changed { address: "https://{switch:a,b}.host/{z}/{x}/{y}@2x.png?map=m".into(), removed: 3 } + ); + assert_eq!(name_class("%2525252574oken", &QueryNames::default()), NameClass::Unresolved); + assert_eq!(name_class(&over, &QueryNames::default()), NameClass::Unresolved); + let mut callback = "https://h.invalid/cb?code=RGSYNTH_code_20261103&%73tate=flow".to_string(); + redact_query(&mut callback); + assert_eq!(callback, "https://h.invalid/cb?%73tate=flow"); + let (clean, count) = scrub_addresses( + "failed https://u:RGSYNTH_user_20261103@h.invalid/a?%74oken=RGSYNTH_query_20261103 then https://h.invalid/b?layer=a%20b\n", + ); + assert_eq!(clean, "failed https://h.invalid/a then https://h.invalid/b?layer=a%20b\n"); + assert_eq!(count, 1); + assert_eq!(scrub_addresses("(https://h.invalid/a?token=RGSYNTH_punctuation_20261103), done").0, "(https://h.invalid/a), done"); + for (text, public, count) in [ + ("read https:/h.invalid/x?token=RGSYNTH_0", "read https:", 1), + ("\t(HTTP:/h.invalid/x?token=RGSYNTH_0), done\n", "\t(HTTP:), done\n", 1), + ("[s3:bucket/x?token=RGSYNTH_0]", "[s3:]", 1), + ("https:x", "https:", 1), + ("s3:", "s3:", 1), + ("read https:/h.invalid/x then https://h.invalid/b?layer=a%20b", "read https: then https://h.invalid/b?layer=a%20b", 1), + ( + "http://h.invalid/a https://h.invalid/b?layer=a%20b s3://bucket/x", + "http://h.invalid/a https://h.invalid/b?layer=a%20b s3://bucket/x", + 0, + ), + ("myhttps:/h.invalid/x?token=RGSYNTH_unrelated_20261104", "myhttps:/h.invalid/x?token=RGSYNTH_unrelated_20261104", 0), + ("dataset token=RGSYNTH_data_20261104 key password", "dataset token=RGSYNTH_data_20261104 key password", 0), + ] { + let (scrubbed, changed) = scrub_addresses(text); + assert_eq!((scrubbed.as_ref(), changed), (public, count), "{text}"); + if count == 0 { + assert!(matches!(scrubbed, Cow::Borrowed(_))); + } + } + assert!(matches!(scrub_addresses("dataset token key password").0, Cow::Borrowed(_))); + assert_eq!(public_web_address("/data/token?key=password"), Public::Unchanged); + assert_eq!(public_web_address("https:///path"), Public::Withheld); + assert_eq!(public_web_address("https://u:RGSYNTH_malformed_20261103/path"), Public::Withheld); + assert_eq!( + public_web_address("https://u:RGSYNTH_malformed_20261103/part@h.invalid/a"), + Public::Changed { address: "https://h.invalid/a".into(), removed: 1 } + ); + } + + #[test] + fn inherited_queries_require_the_reviewed_relationship_and_unambiguous_path() { + let source = "https://h.invalid/svc?token=RGSYNTH_inherit_20261103&k=RGSYNTH_unknown_20261103&collections=roads&f=json"; + let target = "https://h.invalid/svc/tiles/{z}/{y}/{x}"; + let kept = inherit_query(source, target, Relation::Constructed, &["f"]); + assert!(!kept.withheld); + assert!(kept.url.ends_with("?token=RGSYNTH_inherit_20261103&k=RGSYNTH_unknown_20261103&collections=roads")); + for destination in [ + target, + "https://other.invalid/svc/tiles", + "http://h.invalid/svc/tiles", + "https://h.invalid:444/svc/tiles", + "https://u@h.invalid/svc/tiles", + "https://h.invalid/svc-other/tiles", + ] { + let f3 = inherit_query(source, destination, Relation::Advertised, &["f"]); + assert_eq!(f3.url, format!("{destination}?collections=roads")); + assert!(f3.withheld); + if destination != target { + let constructed = inherit_query(source, destination, Relation::Constructed, &["f"]); + assert_eq!(constructed, f3); + } + } + for suffix in + ["../tiles", "./tiles", "%2e%2e/tiles", "%2Ftiles", "%5Ctiles", "\\tiles", "%252e/tiles", "/tiles", "tiles//x", "tiles/\nnext"] + { + let to = format!("https://h.invalid/svc/{suffix}"); + let inherited = inherit_query(source, &to, Relation::Constructed, &["f"]); + assert!(inherited.withheld && !inherited.url.contains("RGSYNTH_"), "{to}"); + } + for name in ["%2525252574oken".to_string(), "a".repeat(MAX_NAME_BYTES + 1)] { + let source = format!("https://h.invalid/svc?{name}=RGSYNTH_unresolved_20261103"); + for relation in [Relation::Constructed, Relation::Advertised] { + assert_eq!(inherit_query(&source, target, relation, &[]), Inherited { url: target.into(), withheld: true }); + } + } + let root = "https://h.invalid/?token=RGSYNTH_root_20261103&collections=x"; + assert_eq!(inherit_query(root, target, Relation::Advertised, &[]).url, format!("{target}?collections=x")); + let reviewed = [ReviewedScope { directory: "https://h.invalid/svc/", whole_origin: false }]; + assert_eq!(inherit_reviewed(source, target, Relation::Advertised, &["f"], &reviewed), kept); + assert!(inherit_reviewed(source, "https://h.invalid/svc-other/a", Relation::Advertised, &["f"], &reviewed).withheld); + let unreviewed_root = [ReviewedScope { directory: "https://h.invalid/", whole_origin: false }]; + assert!(inherit_reviewed(source, target, Relation::Advertised, &["f"], &unreviewed_root).withheld); + assert!(F2_SCOPES.is_empty()); + let to = format!("{target}?%74oken=destination-owned#look"); + let own = inherit_query(source, &to, Relation::Constructed, &["f"]); + assert!(own.url.contains("%74oken=destination-owned") && !own.url.contains("RGSYNTH_inherit")); + assert!(own.url.ends_with("#look")); + assert!(!inherit_query("https://h.invalid/svc?collections=x", target, Relation::Advertised, &[]).withheld); + for key in ["c_r_s", "%FFcollections", "+collections", "f_"] { + let source = format!("https://h.invalid/svc?{key}=RGSYNTH_alias_20261103"); + let inherited = inherit_query(&source, target, Relation::Advertised, &["f"]); + assert_eq!(inherited, Inherited { url: target.into(), withheld: true }); + } + assert_eq!( + inherit_query("https://h.invalid/svc?%63ollections=x&subset-crs=y", target, Relation::Advertised, &[]).url, + format!("{target}?%63ollections=x&subset-crs=y") + ); + let duplicate = inherit_query( + "https://h.invalid/svc?token=source;collections=x", + &format!("{target}?token=destination"), + Relation::Constructed, + &[], + ); + assert_eq!(duplicate.url, format!("{target}?token=destination")); + assert!(duplicate.withheld); + } +} diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index f75553520..b383e88b7 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -491,73 +491,12 @@ fn normalized_key(key: &str) -> impl Iterator + '_ { /// Whether `key` is `target` (lower case, letters and digits) however it is written. fn key_is(key: &str, target: &str) -> bool { - normalized_key(key).eq(target.chars()) -} - -/// A name with its percent and `\u` escapes decoded, to the depth an attacker could nest them. -fn decoded_name(key: &str) -> String { - let mut name = key.to_string(); - for _ in 0..4 { - let bytes = name.as_bytes(); - let mut decoded = String::with_capacity(name.len()); - let mut i = 0; - while let Some(&b) = bytes.get(i) { - if b == b'%' - && let Some(hex) = name.get(i + 1..i + 3) - && let Ok(value) = u8::from_str_radix(hex, 16) - { - decoded.push(char::from(value)); - i += 3; - } else if b == b'\\' - && bytes.get(i + 1) == Some(&b'u') - && let Some(hex) = name.get(i + 2..i + 6) - && let Ok(value) = u32::from_str_radix(hex, 16) - && let Some(c) = char::from_u32(value) - { - decoded.push(c); - i += 6; - } else if let Some(c) = name.get(i..).and_then(|s| s.chars().next()) { - decoded.push(c); - i += c.len_utf8(); - } else { - break; - } - } - if decoded == name { - break; - } - name = decoded; - } - name + crate::address::normalized_name(key).is_some_and(|name| name == target) } +/// Shared bounded name decoding; diagnostics additionally omit log-only fields. fn secret_name(key: &str) -> bool { - // Decode only names, never values: decoding a URL value could introduce whitespace and - // incorrectly end its redacted query. Work and allocation are capped by MAX_INPUT_BYTES, and - // a name with nothing to decode is not copied. - let decoded; - let name = if key.contains(['%', '\\']) { - decoded = decoded_name(key); - decoded.as_str() - } else { - key - }; - let normalized: String = normalized_key(name).collect(); - crate::source::credentials::secret_key(name) - || crate::source::credentials::secret_key(&normalized) - || normalized.contains("credential") - || normalized.contains("connectionstring") - || normalized.contains("privatekey") - || normalized.contains("accountkey") - // A prefixed name is still one: MAPBOX_API_KEY, X_API_KEY, AWS_ACCESS_KEY_ID. - || normalized.contains("apikey") - || normalized.contains("accesskey") - || normalized.contains("sharedaccess") - || normalized.contains("body") - || matches!( - normalized.as_str(), - "basic" | "bearer" | "authentication" | "wwwauthenticate" | "proxyauthenticate" | "userid" | "username" - ) + crate::address::log_sensitive(key) } /// Whether the name is an HTTP credential header: its value is always a scheme and a credential @@ -1246,6 +1185,9 @@ mod tests { assert_eq!(redact("server: B.earer hidden"), "server: [REDACTED]"); assert_eq!(redact("server: Ba-sic dXNlcjpwYXNz"), "server: [REDACTED]"); assert_eq!(redact("pass%77ord=x"), "[REDACTED]"); + for key in ["%2574oken", "%2525252574oken"] { + assert!(!redact(&format!("{key}=RGSYNTH_diagnostic_20261103")).contains("RGSYNTH_")); + } } #[test] diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index cf7e0e9ad..7577644cc 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -4,6 +4,7 @@ //! bad features are skipped and counted in [`ReadReport`]. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod address; pub mod cityjson; pub mod color; mod dbf; diff --git a/crates/data/src/ogc.rs b/crates/data/src/ogc.rs index ae6147b40..540a8579d 100644 --- a/crates/data/src/ogc.rs +++ b/crates/data/src/ogc.rs @@ -442,6 +442,8 @@ pub struct Service { /// Some advertised collections were left out (past the listing limit, or with an items /// address RustGIS doesn't fetch): the listing isn't the whole service. pub incomplete: bool, + /// Destinations that did not inherit one or more query pairs; reported by the caller. + pub withheld_addresses: u32, } /// Most layers listed from one service. @@ -580,7 +582,8 @@ fn query_value<'a>(url: &'a str, key: &str) -> Option<&'a str> { let q = url.split('#').next()?.split_once('?')?.1; q.split('&').find_map(|kv| { let (k, v) = kv.split_once('=').unwrap_or((kv, "")); - k.eq_ignore_ascii_case(key).then_some(v) + (if key == "token" { crate::address::normalized_name(k).as_deref() == Some("token") } else { k.eq_ignore_ascii_case(key) }) + .then_some(v) }) } @@ -2205,7 +2208,8 @@ fn arcgis(url: &str, v: &serde_json::Value) -> Result { max_paged_features() )); } - // A secured service's token goes with every request. + // Offers are constructed under the service. A suffix supplied by its document still has + // to pass the raw-path check before it may receive the service's token. if let Some(token) = query_value(url, "token").filter(|t| !t.is_empty()) { for l in &mut service.layers { match &mut l.offer { @@ -2214,7 +2218,13 @@ fn arcgis(url: &str, v: &serde_json::Value) -> Result { | Offer::Style { url: u } | Offer::VectorTiles { template: u, .. } | Offer::Service { url: u } - | Offer::Coverage(Coverage { endpoint: u, .. }) => *u = with_query(u, &format!("token={token}")), + | Offer::Coverage(Coverage { endpoint: u, .. }) => { + if crate::address::Scope::of(url).is_some_and(|scope| scope.covers_below(u)) { + *u = with_query(u, &format!("token={token}")); + } else { + service.withheld_addresses = service.withheld_addresses.saturating_add(1); + } + } } } } @@ -2328,7 +2338,8 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { service.collections_url = collections_link(url, &links); let name = if title.is_empty() { "tiles".to_string() } else { title }; // The dataset's layers aren't listed here, so its vector tiles can't be styled: map tiles only. - if let Some((t, list)) = tileset(url, &links, TileRel::Map) { + if let Some((t, list, withheld)) = tileset(url, &links, TileRel::Map) { + service.withheld_addresses = service.withheld_addresses.saturating_add(u32::from(withheld)); service.layers.push(ServiceLayer { zooms: None, name: "map".into(), @@ -2382,7 +2393,8 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { // tiles' layer is named after the collection, as the standard has it. let map = tileset(url, &links, TileRel::Map); let vector = tileset(url, &links, TileRel::Vector); - if let Some((t, list)) = &map { + if let Some((t, list, withheld)) = &map { + service.withheld_addresses = service.withheld_addresses.saturating_add(u32::from(*withheld)); service.layers.push(ServiceLayer { zooms: None, name: format!("{id} (map tiles)"), @@ -2395,7 +2407,8 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { legends: Vec::new(), }); } - if let Some((t, list)) = &vector { + if let Some((t, list, withheld)) = &vector { + service.withheld_addresses = service.withheld_addresses.saturating_add(u32::from(*withheld)); service.layers.push(ServiceLayer { zooms: None, name: format!("{id} (vector tiles)"), @@ -2508,9 +2521,9 @@ enum TileRel { /// `tilesets-vector` link: the template at the usual path under the tileset list /// (`…/WebMercatorQuad/{tileMatrix}/{tileRow}/{tileCol}`), and the list's address (JSON), whose /// WebMercatorQuad tileset advertises the template [`resolve_tiles`] reads (the usual path is what -/// is used when that can't be read). Both keep the link's query (a token, a collections choice), -/// all but `f`. -fn tileset(from: &str, links: &[serde_json::Value], which: TileRel) -> Option<(String, String)> { +/// is used when that can't be read). The constructed template keeps the link's query except +/// `f` only under its checked source path; the third result counts a withheld destination. +fn tileset(from: &str, links: &[serde_json::Value], which: TileRel) -> Option<(String, String, bool)> { let suffix = match which { TileRel::Map => "tilesets-map", TileRel::Vector => "tilesets-vector", @@ -2534,23 +2547,12 @@ fn tileset(from: &str, links: &[serde_json::Value], which: TileRel) -> Option<(S TileRel::Vector => format!("{at}/WebMercatorQuad/{{z}}/{{y}}/{{x}}?f=mvt"), }; let list = if query_value(&href, "f").is_some() { href.clone() } else { with_query(&href, "f=json") }; - Some((carry_query(&template, &href), list)) + let inherited = crate::address::inherit_query(&href, &template, crate::address::Relation::Constructed, &["f"]); + Some((inherited.url, list, inherited.withheld)) } -/// `template` with the query parameters of `link` it doesn't have, but `f` (the representation -/// the link was for). -fn carry_query(template: &str, link: &str) -> String { - let q = link.split('#').next().and_then(|u| u.split_once('?')).map(|(_, q)| q).unwrap_or(""); - let mut out = template.to_string(); - for kv in q.split('&').take(32) { - let k = kv.split_once('=').map(|(k, _)| k).unwrap_or(kv); - if kv.is_empty() || k.eq_ignore_ascii_case("f") || query_value(&out, k).is_some() || kv.contains(['{', '}']) { - continue; - } - out = with_query(&out, kv); - } - out -} +/// Notice template shared by native discovery and the translated asynchronous UI resolver. +pub const QUERY_WITHHELD_NOTICE: &str = "{n} address(es) the service listed didn't get its keys or other settings: RustGIS sends those only to addresses it builds itself from the one given; add such an address with its own key if it needs one"; /// What an OGC API tiles document says on the way to a tile template: the template, or the /// address of the document to read next (a tileset list's WebMercatorQuad tileset). @@ -2560,15 +2562,36 @@ pub enum TilesStep { Next(String), } +/// One discovery result and the number of destinations that lost query settings. Callers must +/// report this count, including when a later fetch fails and falls back to a constructed URL. +#[derive(Clone, Debug, PartialEq)] +pub struct TilesResolution { + pub step: TilesStep, + pub withheld_addresses: u32, +} + +/// Applies the advertised-link policy; the document cannot claim a constructed relationship. +fn advertised_tiles(list: &str, to: &str, next: bool) -> TilesResolution { + let inherited = crate::address::inherit_query(list, to, crate::address::Relation::Advertised, &["f"]); + let url = inherited.url; + let step = if next { + TilesStep::Next(if query_value(&url, "f").is_some() { url } else { with_query(&url, "f=json") }) + } else { + TilesStep::Template(url) + }; + TilesResolution { step, withheld_addresses: u32::from(inherited.withheld) } +} + /// One step of reading an OGC API tile template: `body` is the tileset list (or a tileset) -/// fetched from `doc`; `list` is the list's address, whose query the template keeps. `None` when +/// fetched from `doc`; `list` is the query source. Advertised links receive SAFE_OPTIONS only; +/// withheld settings are counted for the caller's notice. `None` when /// the document names no WebMercatorQuad tileset or tile template. -pub fn resolve_tiles(list: &str, doc: &str, body: &[u8], vector: bool) -> Option { +pub fn resolve_tiles(list: &str, doc: &str, body: &[u8], vector: bool) -> Option { let v: serde_json::Value = serde_json::from_slice(body).ok()?; let links = |v: &serde_json::Value| v.get("links").and_then(|l| l.as_array()).cloned().unwrap_or_default(); let Some(sets) = v.get("tilesets").and_then(|t| t.as_array()) else { // A tileset: its tile template. - return item_template(doc, &links(&v), vector).map(|t| TilesStep::Template(carry_query(&t, list))); + return item_template(doc, &links(&v), vector).map(|t| advertised_tiles(list, &t, false)); }; let mercator = |t: &serde_json::Value| { let tiling = links(t); @@ -2592,14 +2615,13 @@ pub fn resolve_tiles(list: &str, doc: &str, body: &[u8], vector: bool) -> Option let t = sets.iter().take(200).find(|t| mercator(t) && kind_fits(t))?; let ls = links(t); if let Some(template) = item_template(doc, &ls, vector) { - return Some(TilesStep::Template(carry_query(&template, list))); + return Some(advertised_tiles(list, &template, false)); } let next = ls .iter() .find(|l| l.get("rel").and_then(|r| r.as_str()).is_some_and(|r| r == "self" || r.ends_with("/tileset"))) .and_then(|l| resolve(doc, l.get("href")?.as_str()?))?; - let next = carry_query(&next, list); - Some(TilesStep::Next(if query_value(&next, "f").is_some() { next } else { with_query(&next, "f=json") })) + Some(advertised_tiles(list, &next, true)) } /// A tileset's advertised tile template (its `item` link, `{tileMatrix}/{tileRow}/{tileCol}`) as @@ -6000,35 +6022,65 @@ mod tests { {"tileMatrixSetURI": "http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldCRS84Quad", "links": [{"rel": "self", "href": "crs84"}]}, {"dataType": "vector", "tileMatrixSetURI": "http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad", "links": [{"rel": "self", "href": "https://x.org/c/roads/tiles/smerc"}]}]}"#; assert_eq!( - resolve_tiles(list, list, sets, true), - Some(TilesStep::Next("https://x.org/c/roads/tiles/smerc?token=abc&f=json".into())) + resolve_tiles(list, list, sets, true).map(|r| r.step), + Some(TilesStep::Next("https://x.org/c/roads/tiles/smerc?f=json".into())) ); - assert_eq!(resolve_tiles(list, list, sets, false), None, "no map tileset in WebMercatorQuad"); - // The tileset's item links: MVT for vector tiles, PNG first for map tiles, the token kept. + assert_eq!(resolve_tiles(list, list, sets, false).map(|r| r.step), None, "no map tileset in WebMercatorQuad"); + // The tileset's item links: MVT for vector tiles, PNG first for map tiles, the list's token withheld under F3. let tileset = br#"{"links": [ {"rel": "item", "type": "image/jpeg", "templated": true, "href": "https://x.org/c/roads/tiles/smerc/{tileMatrix}/{tileRow}/{tileCol}.jpg"}, {"rel": "item", "type": "image/png", "templated": true, "href": "https://x.org/c/roads/tiles/smerc/{tileMatrix}/{tileRow}/{tileCol}.png"}, {"rel": "item", "type": "application/vnd.mapbox-vector-tile", "templated": true, "href": "smerc/{tileMatrix}/{tileRow}/{tileCol}.mvt"}]}"#; let doc = "https://x.org/c/roads/tiles/smerc?token=abc&f=json"; assert_eq!( - resolve_tiles(list, doc, tileset, true), - Some(TilesStep::Template("https://x.org/c/roads/tiles/smerc/{z}/{y}/{x}.mvt?token=abc".into())) + resolve_tiles(list, doc, tileset, true).map(|r| r.step), + Some(TilesStep::Template("https://x.org/c/roads/tiles/smerc/{z}/{y}/{x}.mvt".into())) ); assert_eq!( - resolve_tiles(list, doc, tileset, false), - Some(TilesStep::Template("https://x.org/c/roads/tiles/smerc/{z}/{y}/{x}.png?token=abc".into())) + resolve_tiles(list, doc, tileset, false).map(|r| r.step), + Some(TilesStep::Template("https://x.org/c/roads/tiles/smerc/{z}/{y}/{x}.png".into())) ); // A list that carries its tilesets' item links needs no second document. let inline = br#"{"tilesets": [{"tileMatrixSetId": "WebMercatorQuad", "links": [ {"rel": "item", "type": "image/png", "href": "https://x.org/t/{tileMatrix}/{tileRow}/{tileCol}"}]}]}"#; - assert_eq!(resolve_tiles(list, list, inline, false), Some(TilesStep::Template("https://x.org/t/{z}/{y}/{x}?token=abc".into()))); + assert_eq!( + resolve_tiles(list, list, inline, false).map(|r| r.step), + Some(TilesStep::Template("https://x.org/t/{z}/{y}/{x}".into())) + ); // A template with other variables, or no web address, isn't taken. for bad in [ br#"{"links": [{"rel": "item", "href": "https://x.org/{style}/{tileMatrix}/{tileRow}/{tileCol}"}]}"#.as_slice(), br#"{"links": [{"rel": "item", "href": "javascript:{tileMatrix}"}]}"#.as_slice(), b"not json", ] { - assert_eq!(resolve_tiles(list, doc, bad, false), None); + assert_eq!(resolve_tiles(list, doc, bad, false).map(|r| r.step), None); + } + // F3 is independent of origin and directory containment. Destination-owned keys stay; + // only the allowlisted collection choice may follow from the list. + let list = "https://h.invalid/svc?token=RGSYNTH_list_20261103&k=RGSYNTH_unknown_20261103&collections=roads"; + for host in ["h.invalid", "elsewhere.invalid"] { + let doc = format!( + r#"{{"links":[{{"rel":"item","type":"image/png","href":"https://{host}/svc/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?own=destination"}}]}}"# + ); + let resolved = resolve_tiles(list, list, doc.as_bytes(), false).unwrap(); + assert_eq!(resolved.withheld_addresses, 1); + let TilesStep::Template(template) = resolved.step else { panic!("expected template") }; + assert_eq!(template, format!("https://{host}/svc/{{z}}/{{y}}/{{x}}?own=destination&collections=roads")); + } + let clean = "https://h.invalid/svc?collections=roads"; + assert_eq!(resolve_tiles(clean, clean, tileset, false).unwrap().withheld_addresses, 0); + // Even a suffix appended by RustGIS must pass the raw-path check. The normal style + // stays under its service; an advertised '..' style does not inherit its token. + for (style, withheld) in [("resources/styles", 0), ("../other/styles", 1), ("%2e%2e/other/styles", 1)] { + let description = serde_json::json!({"defaultStyles": style}); + let service = parse( + "https://h.invalid/rest/services/a/VectorTileServer?token=RGSYNTH_style_20261103", + description.to_string().as_bytes(), + ) + .unwrap(); + assert_eq!(service.withheld_addresses, withheld); + let Offer::Style { url } = &service.layers[0].offer else { panic!("expected style") }; + assert_eq!(url.contains("RGSYNTH_style_20261103"), withheld == 0); } } diff --git a/crates/data/src/source/credentials.rs b/crates/data/src/source/credentials.rs index cee9f13b4..e649530f8 100644 --- a/crates/data/src/source/credentials.rs +++ b/crates/data/src/source/credentials.rs @@ -1,97 +1,3 @@ -//! Secret-name and query filtering shared by connection addresses and saved sources. +//! Compatibility path for the shared credential-name and query policy. -/// Whether a key names a secret, wherever it is: an exact list (case aside), and any key -/// naming a password, secret or token. Context-dependent query keys are handled by [`redact_query`]. -pub fn secret_key(k: &str) -> bool { - secret_key_lower(&k.to_ascii_lowercase()) -} - -fn secret_key_lower(k: &str) -> bool { - matches!( - k, - "token" - | "access_token" - | "id_token" - | "refresh_token" - | "api_key" - | "apikey" - | "api-key" - | "x-api-key" - | "key" - | "access_key" - | "accesskey" - | "access_key_id" - | "accesskeyid" - | "secret" - | "client_secret" - | "password" - | "passwd" - | "pwd" - | "sig" - | "signature" - | "sas" - | "authorization" - | "proxy-authorization" - | "cookie" - | "set-cookie" - | "bearer" - ) || k.starts_with("x-amz-") - || k.starts_with("x-goog-") - || k.contains("password") - || k.contains("secret") - || k.contains("token") -} - -/// The query keys of an Azure shared access signature, secret only with both `sig` and `sv`. -const SAS_KEYS: &[&str] = - &["sv", "se", "sp", "sr", "st", "spr", "sip", "si", "ss", "srt", "skoid", "sktid", "skt", "ske", "sks", "skv", "sdd"]; - -fn secret_in(k: &str, keys: &[String]) -> bool { - let has = |x: &str| keys.iter().any(|q| q == x); - secret_key_lower(k) || (has("sig") && has("sv") && SAS_KEYS.contains(&k)) || (k == "code" && has("state")) -} - -/// Removes secret query pairs without changing the address's user-info or connection options. -/// Callers must remove passwords from user-info and fragments before calling this function. -pub fn redact_query(address: &mut String) { - let Some(at) = address.find('?') else { return }; - let query = address.get(at + 1..).unwrap_or(""); - let keys: Vec = query.split('&').map(|kv| kv.split_once('=').map_or(kv, |(k, _)| k).to_ascii_lowercase()).collect(); - if query.split('&').zip(&keys).all(|(kv, key)| !kv.is_empty() && !secret_in(key, &keys)) { - return; - } - let mut kept = String::with_capacity(query.len()); - for (kv, key) in query.split('&').zip(&keys) { - if kv.is_empty() || secret_in(key, &keys) { - continue; - } - if !kept.is_empty() { - kept.push('&'); - } - kept.push_str(kv); - } - address.truncate(at); - if !kept.is_empty() { - address.push('?'); - address.push_str(&kept); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn access_key_ids_and_their_secrets_are_secret_names() { - for k in ["accessKeyId", "access_key_id", "AccessKeyID", "secretAccessKey", "sessionToken", "X-Amz-Credential"] { - assert!(secret_key(k), "{k}"); - } - for k in ["remember", "region", "buckets", "endpoint"] { - assert!(!secret_key(k), "{k}"); - } - // In a query, the key id goes with the signature. - let mut a = "https://s3.example.org/b/k?accessKeyId=AKIDEXAMPLE&prefix=x".to_string(); - redact_query(&mut a); - assert_eq!(a, "https://s3.example.org/b/k?prefix=x"); - } -} +pub use crate::address::{NameClass, QueryNames, log_sensitive, name_class, normalized_name, redact_query, secret_key}; diff --git a/crates/data/src/source/pgconn.rs b/crates/data/src/source/pgconn.rs index 972b41930..0f37e6e96 100644 --- a/crates/data/src/source/pgconn.rs +++ b/crates/data/src/source/pgconn.rs @@ -14,21 +14,19 @@ pub fn public_label(s: &str) -> Option { let t = s.trim(); let lower = t.to_ascii_lowercase(); if lower.starts_with("postgresql://") || lower.starts_with("postgres://") { - let (no_userinfo, _) = split_secrets(t, &["password"]); - // A secret in the query (`?password=…`) goes too. - return Some(match no_userinfo.split_once('?') { - Some((base, q)) => { - let kept: Vec<&str> = q - .split('&') - .filter(|kv| { - let k = kv.split_once('=').map(|(k, _)| k).unwrap_or(kv); - !kv.is_empty() && !SECRET_KEYS.iter().any(|s| k.eq_ignore_ascii_case(s)) - }) - .collect(); - if kept.is_empty() { base.to_string() } else { format!("{base}?{}", kept.join("&")) } - } - None => no_userinfo, - }); + let public = split_secrets(t, SECRET_KEYS).0; + let (_, rest) = public.split_once("://")?; + let authority = rest.split(['/', '?', '#']).next()?; + let hosts = authority.rsplit_once('@').map_or(authority, |(_, hosts)| hosts); + // A non-numeric port with no delimited password is an unreadable authority. Multi-host + // URIs, IPv6 and empty hosts (the local socket) keep libpq's existing spelling. + if hosts.split(',').any(|host| { + let port = if host.starts_with('[') { host.split_once(']').map(|(_, rest)| rest) } else { Some(host) }; + port.and_then(|h| h.split_once(':')).is_some_and(|(_, p)| !p.bytes().all(|b| b.is_ascii_digit())) + }) { + return None; + } + return Some(public); } let rest = ["postgresql:", "postgres:", "pg:"].iter().find(|p| lower.starts_with(**p)).and_then(|p| t.get(p.len()..))?; let (public, _) = split_secrets(rest.trim(), SECRET_KEYS); @@ -122,13 +120,17 @@ fn query_secrets(s: &str, secrets: &[&str]) -> (String, Option) { let mut kept = Vec::new(); for kv in query.split('&').filter(|kv| !kv.is_empty()) { let (k, v) = kv.split_once('=').unwrap_or((kv, "")); - if k.eq_ignore_ascii_case("password") { + if crate::address::normalized_name(k).as_deref() == Some("password") { + // The URI driver decodes the name and password once; keyword strings below are + // intentionally not percent-decoded. password = Some(percent_decode(v)); - } else if !secrets.iter().any(|s| k.eq_ignore_ascii_case(s)) { + } else if !secrets.iter().any(|s| crate::address::normalized_name(k).as_deref() == Some(s)) { kept.push(kv); } } - (if kept.is_empty() { base.to_string() } else { format!("{base}?{}", kept.join("&")) }, password) + let mut public = if kept.is_empty() { base.to_string() } else { format!("{base}?{}", kept.join("&")) }; + crate::address::redact_query(&mut public); + (public, password) } /// A libpq keyword/value connection string's pairs (password included), in order; a value is a @@ -221,5 +223,17 @@ mod tests { assert_eq!(pw.as_deref(), Some("s@c")); let (public, pw) = super::split_secrets("postgresql://h/g?password=x", &["password"]); assert_eq!((public.as_str(), pw.as_deref()), ("postgresql://h/g", Some("x"))); + let uri = "postgresql://u@h.invalid/db?pass%77ord=RGSYNTH_pg_20261103&sslmode=require&pass%66ile=RGSYNTH_file_20261103"; + assert_eq!(super::public_label(uri).as_deref(), Some("postgresql://u@h.invalid/db?sslmode=require")); + let (public, password) = super::query_secrets(uri, super::SECRET_KEYS); + assert_eq!(public, "postgresql://u@h.invalid/db?sslmode=require"); + assert_eq!(password.as_deref(), Some("RGSYNTH_pg_20261103")); + let clean = "postgresql://u@h.invalid/db?sslmode=require"; + assert_eq!(super::public_label(clean).as_deref(), Some(clean)); + assert_eq!(super::public_label("postgresql://u:RGSYNTH_unreadable_20261103/path"), None); + let keyword = "host=h.invalid dbname='a%20b' user=u password='RGSYNTH_keyword_20261103'"; + let (public, password) = super::split_secrets(keyword, super::SECRET_KEYS); + assert_eq!(public, "host=h.invalid dbname=a%20b user=u"); + assert_eq!(password.as_deref(), Some("RGSYNTH_keyword_20261103")); } } diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index c18cf994e..7fd5045ff 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -1080,6 +1080,7 @@ fn queue_file(s: &mut Session, file_name: String, bytes: Vec, record: GpReco Ok(json!({"pending": true, "download": id, "file": file_name})) } +/// Records a completed file delivery through the generated-message history seam. fn file_delivered(s: &mut Session, p: &Value) -> CmdResult { let id = p.get("id").and_then(Value::as_u64).ok_or("id is required")?; let i = s.file_outputs.iter().position(|f| f.id == id).ok_or("no pending file with that id")?; @@ -1099,7 +1100,7 @@ fn file_delivered(s: &mut Session, p: &Value) -> CmdResult { s.notify(message.clone(), !file.record.ok); } let out = json!({"file": file.file_name, "delivered": file.record.ok, "messages": file.record.messages}); - s.project.gp_history.push(file.record); + s.push_gp_record(file.record); if s.project.gp_history.len() > 500 { s.project.gp_history.remove(0); } @@ -1191,7 +1192,7 @@ fn apply(s: &mut Session, ran: Ran, params: Value, res: Result { - s.project.gp_history.push(GpRecord { + s.push_gp_record(GpRecord { tool: name, id, params, @@ -1223,7 +1224,7 @@ fn apply(s: &mut Session, ran: Ran, params: Value, res: Result Vec::new(), }; let outputs = ids.iter().filter_map(|id| s.map().layer(*id).map(|l| l.name.clone())).collect(); - s.project.gp_history.push(GpRecord { tool: name, id, params, command, at, ok: true, messages: result.messages, millis, outputs }); + s.push_gp_record(GpRecord { tool: name, id, params, command, at, ok: true, messages: result.messages, millis, outputs }); if s.project.gp_history.len() > 500 { s.project.gp_history.remove(0); } diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 3a22bd151..aa3fab750 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -20,64 +20,75 @@ pub fn commands() -> Vec { ] } +/// Formats a replayable code fragment inside the guarded history projection. fn code(s: &mut Session, p: &Value) -> CmdResult { let i = uint(p, "run").and_then(|i| usize::try_from(i).ok()).ok_or("run is required: a run's place in the history, from 0")?; - let r = s.project.gp_history.get(i).ok_or("no run there in the history")?; - let (cmd, params) = replay(r)?; - let text = match ps(p, "as").unwrap_or("python") { - "python" => python(&cmd, ¶ms), - "command" => format!("{cmd} {params}"), - "cli" => format!("rustgis-cli run --project {} --cmd {}", shell(&project_file(s)), shell(&format!("{cmd}={params}"))), - other => return Err(format!("unknown kind '{other}': python, command or cli")), - }; + let project = project_file(s); + let r = s.project.gp_history.get_mut(i).ok_or("no run there in the history")?; + let (text, report) = crate::durable::output(r, None, |r| { + let (cmd, params) = replay_written(r)?; + match ps(p, "as").unwrap_or("python") { + "python" => Ok(python(&cmd, ¶ms)), + "command" => Ok(format!("{cmd} {params}")), + "cli" => Ok(format!("rustgis-cli run --project {} --cmd {}", shell(&project), shell(&format!("{cmd}={params}")))), + other => Err(format!("unknown kind '{other}': python, command or cli")), + } + })?; + crate::durable::notify(s, &report); Ok(json!({"code": text})) } /// The history as a Python script that opens the project and runs each tool again, in order. A /// run that failed, or can't be run again, is there as a comment. fn script(s: &mut Session, _: &Value) -> CmdResult { - let runs = &s.project.gp_history; - if runs.is_empty() { + if s.project.gp_history.is_empty() { return Err("the history is empty: run a tool first".into()); } - let name = &s.project.name; - let mut text = format!( - "# The geoprocessing history of the RustGIS project {}, run again in order.\n\ + let name = s.project.name.clone(); + let project = project_file(s); + let unsaved = s.path.is_none(); + let dirty = s.dirty(); + let (result, report) = crate::durable::output(&mut s.project.gp_history, None, |runs| { + let mut text = format!( + "# The geoprocessing history of the RustGIS project {}, run again in order.\n\ # Layers are named as they were then, so this opens the project they belong to.\n", - quoted(name) - ); - // The script opens the saved file: a layer added since, which a run may have used, isn't - // there until the project is saved again. - if s.path.is_none() { - text.push_str("# The project hasn't been saved yet: save it as this file first.\n"); - } else if s.dirty() { - text.push_str("# The project has changes that aren't saved: save it first, so its layers are there.\n"); - } - text.push_str(&format!("import rustgis\n\ns = rustgis.Session({})\n", quoted(&project_file(s)))); - for (n, r) in runs.iter().enumerate() { - let when = if r.at > 0 { - let (date, time) = rustgis_map::graphics::civil(r.at); - format!(" ({date} {time} UTC)") - } else { - String::new() - }; - text.push_str(&format!("\n# {}. {}{when}", n + 1, line(&r.tool))); - match replay(r) { - Ok((cmd, params)) if r.ok => text.push_str(&format!("\n{}\n", python(&cmd, ¶ms))), - Ok((cmd, params)) => { - // An error may quote a connection, password and all: redacted as a parameter's - // value is, not for its addresses only. - let why = r.messages.first().map(|m| line(&redacted_text(m))).unwrap_or_default(); - text.push_str(&format!(": failed when it ran ({why})\n")); - for l in python(&cmd, ¶ms).lines() { - text.push_str(&format!("# {l}\n")); + quoted(&name) + ); + // The script opens the saved file: a layer added since, which a run may have used, isn't + // there until the project is saved again. + if unsaved { + text.push_str("# The project hasn't been saved yet: save it as this file first.\n"); + } else if dirty { + text.push_str("# The project has changes that aren't saved: save it first, so its layers are there.\n"); + } + text.push_str(&format!("import rustgis\n\ns = rustgis.Session({})\n", quoted(&project))); + for (n, r) in runs.iter().enumerate() { + let when = if r.at > 0 { + let (date, time) = rustgis_map::graphics::civil(r.at); + format!(" ({date} {time} UTC)") + } else { + String::new() + }; + text.push_str(&format!("\n# {}. {}{when}", n + 1, line(&r.tool))); + match replay_written(r) { + Ok((cmd, params)) if r.ok => text.push_str(&format!("\n{}\n", python(&cmd, ¶ms))), + Ok((cmd, params)) => { + // An error may quote a connection, password and all: redacted as a parameter's + // value is, not for its addresses only. + let why = r.messages.first().map(|m| line(&redacted_text(m))).unwrap_or_default(); + text.push_str(&format!(": failed when it ran ({why})\n")); + for l in python(&cmd, ¶ms).lines() { + text.push_str(&format!("# {l}\n")); + } } + Err(e) => text.push_str(&format!(": {}\n", line(&e))), } - Err(e) => text.push_str(&format!(": {}\n", line(&e))), } - } - let file = format!("{}_history.py", file_stem(name)); - Ok(json!({"fileName": file, "text": text})) + let file = format!("{}_history.py", file_stem(&name)); + Ok(json!({"fileName": file, "text": text})) + })?; + crate::durable::notify(s, &report); + Ok(result) } /// The commands other than `gp.run` a run is copied as. The history comes from the project file, @@ -88,14 +99,21 @@ const REPLAYED: &[&str] = &["gp.bufferPoint"]; /// The command that runs `r` again, with its parameters: `gp.run` with the tool, or the command /// that ran it (a point buffered where the map was clicked, `gp.bufferPoint`). pub fn replay(r: &GpRecord) -> Result<(String, Value), String> { + let (id, params) = replay_written(r)?; + let (clean, _) = crate::roles::command(&id, ¶ms, 0); + Ok((id, clean.unwrap_or(params))) +} + +/// Formats a record already inside the durable output transaction; data stays replayable. +fn replay_written(r: &GpRecord) -> Result<(String, Value), String> { if let Some((cmd, params)) = &r.command { let Some(cmd) = REPLAYED.iter().find(|c| **c == cmd.as_str()) else { return Err(format!("{} can't be run again: the history names a command that isn't a geoprocessing one", r.tool)); }; - return Ok((cmd.to_string(), redacted(params))); + return Ok((cmd.to_string(), params.clone())); } let id = tool_of(r).ok_or_else(|| format!("{} can't be run again: the history doesn't say which tool it was", r.tool))?; - Ok(("gp.run".into(), json!({"tool": id, "params": redacted(&r.params)}))) + Ok(("gp.run".into(), json!({"tool": id, "params": r.params}))) } /// The id of the tool `r` ran with `gp.run`: kept with the run, or, in projects from before it diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 4bbd79257..82df1221e 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -517,15 +517,20 @@ fn import_styles(s: &mut Session, p: &Value) -> CmdResult { Ok(json!({"imported": n})) } -/// The project without its features (layer schemas, symbology, layouts and styles stay). +/// The project without features or sources that would reload them; schemas and styles stay. fn template(p: &Project) -> Project { let mut t = p.clone(); t.gp_history.clear(); + t.offline_areas.clear(); for m in &mut t.maps { for l in &mut m.layers { l.dataset.features.clear(); l.selection.clear(); l.dataset.source = None; + l.source = None; + if let Some(archive) = &mut l.design.archive { + archive.clear(); + } // A feature-less template has no loaded rows to save back to a source. l.postgis = None; l.gpkg = None; @@ -540,29 +545,30 @@ fn template(p: &Project) -> Project { } for t in &mut t.tables { t.features.clear(); + t.source = None; } t } /// Packages only completed imports; runtime download reservations cannot be reopened. /// Refuses JSON beyond the project or ZIP inflation limit before creating the archive. -fn package_project(s: &Session) -> Result, String> { +fn package_project(s: &mut Session) -> Result, String> { s.project.ensure_imports_complete()?; let mut p = s.project.clone(); let mut budget = MAX_EMBED; for m in &mut p.maps { *m = embedded_map(m, &mut budget); } - crate::project::strip_addresses(p.addresses_mut()); - let text = serde_json::to_string(&p).map_err(|e| e.to_string())?; + let name = p.name.clone(); + let (text, report) = crate::durable::output_owned(p, |p| serde_json::to_string(p).map_err(|e| e.to_string()))?; + crate::durable::notify(s, &report); let limit = crate::project::MAX_BYTES.min(rustgis_data::zip::MAX_TOTAL as u64); if text.len() as u64 > limit { return Err(format!("the project is too large to package (maximum {} MiB)", limit >> 20)); } - let readme = - format!("{} — a RustGIS project package.\nOpen it with RustGIS (Project › Open), or unzip it for {}.rgis.\n", p.name, p.name); + let readme = format!("{} — a RustGIS project package.\nOpen it with RustGIS (Project › Open), or unzip it for {}.rgis.\n", name, name); rustgis_data::zip::write(&[ - (format!("{}.{}", safe(&p.name), crate::project::EXTENSION), text.into_bytes()), + (format!("{}.{}", safe(&name), crate::project::EXTENSION), text.into_bytes()), ("README.txt".into(), readme.into_bytes()), ]) } @@ -573,7 +579,7 @@ fn safe(name: &str) -> String { } /// A package of the active map; `mobile` drops hidden layers and simplifies geometry. -fn package_map(s: &Session, mobile: bool, tol: f64) -> Result, String> { +fn package_map(s: &mut Session, mobile: bool, tol: f64) -> Result, String> { let mut m = embedded_map(s.map(), &mut MAX_EMBED.clone()); if mobile { // Groups folded into their layers: a hidden group's layers are dropped with the others. @@ -600,19 +606,23 @@ fn package_map(s: &Session, mobile: bool, tol: f64) -> Result, String> { } } crate::project::ensure_map_imports_complete(&m)?; - crate::project::strip_addresses(m.addresses_mut()); - let text = serde_json::to_string(&m).map_err(|e| e.to_string())?; + let name = m.name.clone(); + let (text, report) = crate::durable::output_owned(m, |m| serde_json::to_string(m).map_err(|e| e.to_string()))?; + crate::durable::notify(s, &report); let kind = if mobile { "mobile map package" } else { "map package" }; - let readme = format!("{} — a RustGIS {kind}.\nAdd it to a project with Data › Import.\n", m.name); - rustgis_data::zip::write(&[(format!("{}.rgismap", safe(&m.name)), text.into_bytes()), ("README.txt".into(), readme.into_bytes())]) + let readme = format!("{} — a RustGIS {kind}.\nAdd it to a project with Data › Import.\n", name); + rustgis_data::zip::write(&[(format!("{}.rgismap", safe(&name)), text.into_bytes()), ("README.txt".into(), readme.into_bytes())]) } /// The geoprocessing history with the layers its runs made, as GeoJSON. -fn package_gp(s: &Session) -> Result, String> { +fn package_gp(s: &mut Session) -> Result, String> { if s.project.gp_history.is_empty() { return Err("no geoprocessing has been run yet".into()); } - let mut entries = vec![("history.json".to_string(), serde_json::to_vec_pretty(&s.project.gp_history).map_err(|e| e.to_string())?)]; + let (history, report) = + crate::durable::output(&mut s.project.gp_history, None, |h| serde_json::to_vec_pretty(h).map_err(|e| e.to_string()))?; + crate::durable::notify(s, &report); + let mut entries = vec![("history.json".to_string(), history)]; let mut layers = std::collections::HashSet::new(); let mut files = std::collections::HashSet::new(); for rec in s.project.gp_history.iter().filter(|r| r.ok) { @@ -1019,9 +1029,9 @@ pub fn item_bytes(s: &mut Session, kind: &str, p: &Value) -> Result<(String, Vec None => s.map(), }; crate::project::ensure_map_imports_complete(map)?; - let mut map = embedded_map(map, &mut MAX_EMBED.clone()); - crate::project::strip_addresses(map.addresses_mut()); - (map.name.clone(), to_json(&map)?) + let map = embedded_map(map, &mut MAX_EMBED.clone()); + let name = map.name.clone(); + (name, durable_bytes(s, map)?) } "layer" => { let id = s.layer_id(p)?; @@ -1029,9 +1039,9 @@ pub fn item_bytes(s: &mut Session, kind: &str, p: &Value) -> Result<(String, Vec if l.dataset.source.as_deref().is_some_and(|source| source.starts_with("geolibre-pending:")) { return Err("this layer is still downloading; wait for it to finish before saving".into()); } - let mut layer = embed_layer(l, &mut MAX_EMBED.clone()); - crate::project::strip_addresses(layer.addresses_mut().collect()); - (layer.name.clone(), to_json(&layer)?) + let layer = embed_layer(l, &mut MAX_EMBED.clone()); + let name = layer.name.clone(); + (name, durable_bytes(s, layer)?) } "styles" => { if s.project.styles.is_empty() { @@ -1051,7 +1061,7 @@ pub fn item_bytes(s: &mut Session, kind: &str, p: &Value) -> Result<(String, Vec if let Some(n) = ps(p, "name").map(str::trim).filter(|n| !n.is_empty()) { t.name = n.chars().take(200).collect(); } - (t.name.clone(), to_json(&t)?) + (t.name.clone(), durable_bytes(s, t)?) } "projectPackage" => (s.project.name.clone(), package_project(s)?), "mapPackage" | "mobilePackage" => { @@ -1061,8 +1071,11 @@ pub fn item_bytes(s: &mut Session, kind: &str, p: &Value) -> Result<(String, Vec "gpPackage" => ("results".to_string(), package_gp(s)?), "bookmarks" => (format!("{} bookmarks", s.map().name), super::navigate::bookmarks_file(s.map())?), "task" => { - let t = task_ref(s, p)?; - (t.name.clone(), to_json(t)?) + let t = task_mut(s, p)?; + let name = t.name.clone(); + let (bytes, report) = crate::durable::output(t, None, to_json)?; + crate::durable::notify(s, &report); + (name, bytes) } "obj" => { let id = s.layer_id(p)?; @@ -1114,3 +1127,10 @@ pub(crate) fn save_item(s: &mut Session, kind: &str, p: &Value) -> CmdResult { rustgis_data::io::write_atomic(Path::new(&path), &bytes).map_err(|e| format!("{path}: {e}"))?; Ok(json!({"path": path, "bytes": bytes.len()})) } + +/// Consumes the existing item draft, emitting losses after serialization has completed. +fn durable_bytes(s: &mut Session, value: T) -> Result, String> { + let (bytes, report) = crate::durable::output_owned(value, to_json)?; + crate::durable::notify(s, &report); + Ok(bytes) +} diff --git a/crates/engine/src/cmd/notebook.rs b/crates/engine/src/cmd/notebook.rs index b9d99e894..cbe7abf8c 100644 --- a/crates/engine/src/cmd/notebook.rs +++ b/crates/engine/src/cmd/notebook.rs @@ -278,6 +278,7 @@ fn run(s: &mut Session, p: &Value) -> CmdResult { r } +/// Runs command cells and cleans generated output before applying its retained-size bound. fn run_cells(s: &mut Session, p: &Value) -> CmdResult { let i = index(s, p)?; let cells: Vec = match p.get("cell") { @@ -328,8 +329,9 @@ fn run_cells(s: &mut Session, p: &Value) -> CmdResult { .collect(); if lines.is_empty() { "(nothing to run)".to_string() } else { lines.join("\n") } } - Err(e) => e, + Err(e) => rustgis_data::diagnostics::Redacted::new(&e).into_string(), }; + let text = rustgis_data::address::scrub_addresses(&text).0.into_owned(); // Into the cell that ran, which a command removing or adding cells may have moved: the // cell with its source nearest to where it was. let ran_cell = find(s).and_then(|i| s.project.notebooks.get_mut(i)).and_then(|nb| { @@ -578,8 +580,58 @@ fn jupyter_file(nb: &Notebook) -> Value { } /// The notebook as a RustGIS notebook or (nbformat 4) Jupyter notebook. -pub fn notebook_file(s: &Session, i: usize, jupyter: bool) -> Result, String> { - let nb = s.project.notebooks.get(i).ok_or("no notebook")?; - let v = if jupyter { jupyter_file(nb) } else { serde_json::to_value(nb).map_err(|e| e.to_string())? }; - serde_json::to_vec_pretty(&v).map_err(|e| e.to_string()) +pub fn notebook_file(s: &mut Session, i: usize, jupyter: bool) -> Result, String> { + let nb = s.project.notebooks.get_mut(i).ok_or("no notebook")?; + let (bytes, report) = crate::durable::output(nb, None, |nb| { + let v = if jupyter { jupyter_file(nb) } else { serde_json::to_value(nb).map_err(|e| e.to_string())? }; + serde_json::to_vec_pretty(&v).map_err(|e| e.to_string()) + })?; + crate::durable::notify(s, &report); + Ok(bytes) +} + +impl crate::durable::Durable for Notebook { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut crate::durable::Prepare<'_>) { + use crate::durable::Leaf; + let Self { name: _, cells, jupyter: _ } = self; + let commands = self.language().is_none(); + for (i, Cell { kind, source, output, error: _, jupyter: _ }) in cells.iter().enumerate() { + if commands && *kind == CellKind::Code { + p.script(Leaf::CellSource(i), source); + } + if let Some(output) = output { + p.generated(Leaf::CellOutput(i), output); + } + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[crate::durable::Owner], leaf: &crate::durable::Leaf) -> Option> { + use crate::durable::{FieldMut, Leaf}; + if !owners.is_empty() { + return None; + } + match leaf { + Leaf::CellSource(i) => Some(FieldMut::Text(&mut self.cells.get_mut(*i)?.source)), + Leaf::CellOutput(i) => self.cells.get_mut(*i)?.output.as_mut().map(FieldMut::Text), + _ => None, + } + } +} + +/// Cleans a notebook document on its parsed copy, preserving foreign kernels and opaque metadata. +pub(crate) fn durable_document(text: &str, depth: usize) -> Result<(String, crate::durable::Report), String> { + let nb = parse(text, "Notebook")?; + let jupyter = serde_json::from_str::(text).is_ok_and(|v| v.get("nbformat").is_some()); + crate::durable::output_owned_changed(nb, depth, |nb, changed| { + if !changed { + return Ok(text.into()); + } + if jupyter { + serde_json::to_string(&jupyter_file(nb)).map_err(|e| e.to_string()) + } else { + serde_json::to_string(nb).map_err(|e| e.to_string()) + } + }) } diff --git a/crates/engine/src/cmd/presentation.rs b/crates/engine/src/cmd/presentation.rs index 8b59ace7f..16d9e3d1c 100644 --- a/crates/engine/src/cmd/presentation.rs +++ b/crates/engine/src/cmd/presentation.rs @@ -22,6 +22,7 @@ fn need_presentation(s: &Session) -> Option<&'static str> { if s.project.presentations.is_empty() { Some("add a presentation first (Publish › New Presentation)") } else { None } } +/// Registers presentation actions, with public generated video labels and guarded outputs. pub fn commands() -> Vec { vec![ C::new("presentation.new", "New Presentation", "Publish › Presentation", |s, p| { @@ -116,7 +117,7 @@ pub fn commands() -> Vec { (None, None) => None, }; let caption: String = ps(p, "caption").unwrap_or("").chars().take(MAX_CAPTION).collect(); - let title = ps(p, "title").map(str::to_string).unwrap_or_else(|| file_name(&source)); + let title = ps(p, "title").map(str::to_string).unwrap_or_else(|| video_label(&source)); add_page(s, p, title, PageKind::Video { source, poster, caption }) }) .when(need_presentation) @@ -166,8 +167,26 @@ pub fn commands() -> Vec { ] } -fn file_name(source: &str) -> String { - source.rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or(source).chars().take(200).collect() +/// Derives generated labels from the public address before extracting a path or host. User +/// titles/captions and local filenames (including '?' and '#') are data and retain their bytes. +pub(crate) fn video_label(source: &str) -> String { + use crate::durable::{Kept, public_address}; + let clean = match public_address(source) { + Kept::Unchanged => source.to_string(), + Kept::Changed(s) => s, + Kept::Withheld(_) => return "Video".into(), + }; + let inner = crate::sources::template_archive(&clean).map_or(clean.as_str(), |(_, a)| a); + let inner = rustgis_data::s3_address::parse(inner).ok().flatten().map_or(inner, |b| b.url); + let inner = inner.strip_prefix(crate::render::rustgis_tiles::s3::ANONYMOUS).unwrap_or(inner); + let remote = inner.split_once("://").filter(|(s, _)| ["http", "https", "s3"].iter().any(|p| p.eq_ignore_ascii_case(s))); + let name = if let Some((_, rest)) = remote { + let rest = rest.split(['?', '#']).next().unwrap_or(""); + rest.rsplit('/').find(|s| !s.is_empty()).unwrap_or("Video") + } else { + inner.rsplit(['/', '\\']).next().filter(|s| !s.is_empty()).unwrap_or(inner) + }; + name.chars().take(200).collect() } /// The presentation named or numbered by `presentation`, else the active one, else the last. @@ -268,9 +287,11 @@ fn update_page(s: &mut Session, p: &Value) -> CmdResult { } /// A presentation file. -pub fn presentation_file(s: &Session, i: usize) -> Result, String> { - let pres = s.project.presentations.get(i).ok_or("no presentation")?; - serde_json::to_vec_pretty(pres).map_err(|e| e.to_string()) +pub fn presentation_file(s: &mut Session, i: usize) -> Result, String> { + let pres = s.project.presentations.get_mut(i).ok_or("no presentation")?; + let (bytes, report) = crate::durable::output(pres, None, |p| serde_json::to_vec_pretty(p).map_err(|e| e.to_string()))?; + crate::durable::notify(s, &report); + Ok(bytes) } /// A page drawn as a page layout: its title across the top and its content below. @@ -342,7 +363,7 @@ pub fn page_layout(s: &Session, pres: &Presentation, page: &Page) -> Layout { l.add( "Play", (m + bw * 0.3, top + bh / 2.0 - 6.0, bw * 0.4, 12.0), - text(&format!("Video: {}", file_name(source)), 18.0, dark, true), + text(&format!("Video: {}", video_label(source)), 18.0, dark, true), ); } if cap_h > 0.0 { @@ -359,7 +380,7 @@ pub fn page_layout(s: &Session, pres: &Presentation, page: &Page) -> Layout { l.add( "Play", (m + bw * 0.3, top + bh / 2.0 - 6.0, bw * 0.4, 12.0), - text(&format!("Video: {}", file_name(source)), 20.0, Rgba::rgb(0xFF, 0xFF, 0xFF), true), + text(&format!("Video: {}", video_label(source)), 20.0, Rgba::rgb(0xFF, 0xFF, 0xFF), true), ); if !caption.trim().is_empty() { l.add("Caption", (m, h - m - h * 0.07, w - 2.0 * m, h * 0.07), text(caption, 15.0, Rgba::rgb(0x55, 0x5B, 0x66), false)); @@ -430,6 +451,7 @@ pub fn export_bytes(s: &mut Session, i: usize, format: &str, dpi: f64) -> Result } let mut pdf_pages = Vec::new(); let mut slides = String::new(); + let mut output_report = crate::durable::Report::default(); let mut held = 0usize; let (mut frames_pictured, mut pages_pictured) = (Vec::new(), Vec::new()); for k in 0..n { @@ -452,11 +474,25 @@ pub fn export_bytes(s: &mut Session, i: usize, format: &str, dpi: f64) -> Result let body = match &page.kind { // The poster is the drawn page, which already shows the title and caption. The // source is played as given (a file next to the slideshow or a web address). - PageKind::Video { source, .. } => format!( - "", - html_escape(&page.title), - html_escape(playable(source)) - ), + PageKind::Video { source, .. } => { + let source = match crate::durable::public_address(source) { + crate::durable::Kept::Unchanged => source.clone(), + crate::durable::Kept::Changed(s) => { + output_report.videos = output_report.videos.saturating_add(1); + s + } + crate::durable::Kept::Withheld(prefix) => { + output_report.videos = output_report.videos.saturating_add(1); + output_report.withheld = output_report.withheld.saturating_add(1); + prefix.into() + } + }; + format!( + "", + html_escape(&page.title), + html_escape(playable(&source)) + ) + } _ => format!("\"{}\"", html_escape(&page.title)), }; let notes = if page.notes.is_empty() { String::new() } else { format!("", html_escape(&page.notes)) }; @@ -466,6 +502,7 @@ pub fn export_bytes(s: &mut Session, i: usize, format: &str, dpi: f64) -> Result return Err(format!("the export passed {} MB at page {}; export at a lower dpi", BUDGET >> 20, k + 1)); } } + crate::durable::notify(s, &output_report); if format == "pdf" { super::layout::pictured_notices(s, &frames_pictured, &pages_pictured); return Ok((rustgis_render::pdf::write(&pdf_pages, &name), n)); diff --git a/crates/engine/src/cmd/project.rs b/crates/engine/src/cmd/project.rs index 99a5d38c5..7577b233b 100644 --- a/crates/engine/src/cmd/project.rs +++ b/crates/engine/src/cmd/project.rs @@ -148,7 +148,9 @@ pub(crate) fn reset_to(s: &mut Session, p: Project) { reset(s, p); } +/// Replaces project authority, clearing the exchange record, item trust and active connections. fn reset(s: &mut Session, p: Project) { + s.exchange = None; super::connections::clear_active(); s.database_logins.clear_activations(); s.database_logins.clear_requests(); @@ -302,13 +304,12 @@ fn open_text(s: &mut Session, p: &Value) -> CmdResult { fn write(s: &mut Session, path: PathBuf) -> CmdResult { let path = if path.extension().is_none() { path.with_extension(crate::project::EXTENSION) } else { path }; let destination = std::path::absolute(&path).map_err(|error| format!("{}: {error}", path.display()))?; - let stripped = s.project.save(&destination)?; + let report = s.project.save(&destination)?; + let stripped = report.layer_addresses(); s.mark_saved(); s.edit_counts.clear(); s.notify(format!("Saved {}", path.display()), false); - if stripped > 0 { - s.notify(format!("{stripped} layer address(es) were saved without their keys, tokens or passwords: after RustGIS closes, add those layers again to give them"), false); - } + crate::durable::notify(s, &report); s.path = Some(destination); Ok(json!({"path": path.display().to_string(), "stripped": stripped})) } diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index 88fcda57f..9c5478939 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -55,9 +55,9 @@ fn url_of(p: &Value) -> Result<&str, String> { pub(crate) fn without_token(p: &Value) -> Result, String> { use crate::render::rustgis_tiles::arcgis_auth; let Some(url) = ps(p, "url").map(str::trim) else { return Ok(None) }; - let has_token = url - .split_once('?') - .is_some_and(|(_, q)| q.split('&').any(|kv| kv.split('=').next().is_some_and(|k| k.eq_ignore_ascii_case("token")))); + let has_token = url.split_once('?').is_some_and(|(_, q)| { + q.split('&').any(|kv| kv.split('=').next().is_some_and(|k| rustgis_data::address::normalized_name(k).as_deref() == Some("token"))) + }); // ArcGIS REST addresses only: another kind of service's `token` stays in its address. let path = url.split(['?', '#']).next().unwrap_or(url).to_ascii_lowercase(); let arcgis = match ps(p, "type") { @@ -169,6 +169,7 @@ fn capabilities(p: &Value) -> Result { result.service.layers.push(layer); result.sources.push(address.clone()); } + result.service.withheld_addresses = result.service.withheld_addresses.saturating_add(service.withheld_addresses); result.service.notes.extend(service.notes); let features = result.service.layers.iter().any(|l| matches!(l.offer, Offer::Features { .. })); let tiles = result.service.layers.iter().any(|l| matches!(l.offer, Offer::Tiles { .. } | Offer::VectorTiles { .. })); @@ -330,13 +331,15 @@ fn checked_template(service: &str, t: &str) -> Result { /// The tile template an OGC API tileset list advertises (its WebMercatorQuad tileset's), read now /// (the list, then the tileset when the list doesn't carry it). `None` when it can't be read. #[cfg(not(target_arch = "wasm32"))] -fn advertised_template(list: &str, vector: bool) -> Option { +fn advertised_template(list: &str, vector: bool, withheld: &mut u32) -> Option { use rustgis_data::ogc::{TilesStep, resolve_tiles}; let mut doc = list.to_string(); for _ in 0..2 { crate::render::rustgis_tiles::address::check(&doc, Some(list)).ok()?; let body = crate::render::rustgis_tiles::cache::download_blocking(&doc).ok()?; - match resolve_tiles(list, &doc, &body, vector)? { + let resolved = resolve_tiles(list, &doc, &body, vector)?; + *withheld = withheld.saturating_add(resolved.withheld_addresses); + match resolved.step { TilesStep::Template(t) => return checked_template(list, &t).ok(), TilesStep::Next(next) => doc = next, } @@ -346,7 +349,7 @@ fn advertised_template(list: &str, vector: bool) -> Option { /// In the browser nothing waits: the app reads the template itself and passes it as `"tiles"`. #[cfg(target_arch = "wasm32")] -fn advertised_template(_: &str, _: bool) -> Option { +fn advertised_template(_: &str, _: bool, _: &mut u32) -> Option { None } @@ -384,13 +387,18 @@ pub fn service_json(svc: &Service) -> Value { layer }) .collect(); - json!({"kind": svc.kind, "title": svc.title, "count": layers.len(), "layers": layers, "notes": svc.notes}) + let mut notes = svc.notes.clone(); + if svc.withheld_addresses > 0 { + notes.push(rustgis_data::ogc::QUERY_WITHHELD_NOTICE.replace("{n}", &svc.withheld_addresses.to_string())); + } + json!({"kind": svc.kind, "title": svc.title, "count": layers.len(), "layers": layers, "notes": notes}) } -pub(crate) fn layers(_s: &mut Session, p: &Value) -> CmdResult { +pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult { let owned = without_token(p)?; let p = owned.as_ref().unwrap_or(p); let found = capabilities(p)?; + notify_withheld(s, found.service.withheld_addresses); let mut out = service_json(&found.service); if let Some(layers) = out.get_mut("layers").and_then(Value::as_array_mut) { for (layer, source) in layers.iter_mut().zip(found.sources) { @@ -403,6 +411,11 @@ pub(crate) fn layers(_s: &mut Session, p: &Value) -> CmdResult { Ok(out) } +/// Counts destinations, not pairs; clean discovery emits no credential-forwarding notice. +fn notify_withheld(s: &mut Session, count: u32) { + s.notify_loss(rustgis_data::ogc::QUERY_WITHHELD_NOTICE, count); +} + /// Adds one of a service's layers: map layers as tiles (an overlay), a style as a style, features /// downloaded. pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { @@ -411,6 +424,15 @@ pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { let url = url_of(p)?.to_string(); let want = ps(p, "layer").map(str::trim).filter(|l| !l.is_empty()).ok_or("layer is required")?; let found = capabilities(p)?; + let mut withheld = found.service.withheld_addresses; + let result = add_discovered(s, p, &url, want, &found, &mut withheld); + notify_withheld(s, withheld); + result +} + +/// Adds from an already-read listing, accumulating resolver losses for `add`'s single notice +/// epilogue. Validation and execution errors return unchanged through that epilogue too. +fn add_discovered(s: &mut Session, p: &Value, url: &str, want: &str, found: &Discovery, withheld: &mut u32) -> CmdResult { let candidates = || found.service.layers.iter().zip(&found.sources).filter(|(_, source)| ps(p, "source_url").is_none_or(|s| s == source.as_str())); let (layer, source) = candidates() @@ -480,7 +502,7 @@ pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { // the usual path. let advertised = match (&layer.tilesets, ps(p, "tiles")) { (Some(_), Some(t)) => Some(checked_template(source, t)?), - (Some(list), None) => advertised_template(list, matches!(layer.offer, Offer::VectorTiles { .. })), + (Some(list), None) => advertised_template(list, matches!(layer.offer, Offer::VectorTiles { .. }), withheld), _ => None, }; match &layer.offer { diff --git a/crates/engine/src/cmd/sharing.rs b/crates/engine/src/cmd/sharing.rs index c857c4eb6..0b68baa00 100644 --- a/crates/engine/src/cmd/sharing.rs +++ b/crates/engine/src/cmd/sharing.rs @@ -204,25 +204,30 @@ pub fn share_bytes(s: &mut Session, kind: &str, p: &Value) -> Result<(String, Ve Ok((m.name.clone(), ogc_zip(&m.name, &layers, &[])?, m.name.clone())) } "webTool" => { - let ms = &s.project.models; + let ms = &mut s.project.models; let m = match ps(p, "model") { - Some(n) => ms.iter().find(|m| m.name.eq_ignore_ascii_case(n)).ok_or_else(|| format!("no model '{n}'"))?, - None => ms.last().ok_or("make a model first (ModelBuilder)")?, + Some(n) => ms.iter_mut().find(|m| m.name.eq_ignore_ascii_case(n)).ok_or_else(|| format!("no model '{n}'"))?, + None => ms.last_mut().ok_or("make a model first (ModelBuilder)")?, }; - let params: Vec = m - .steps - .iter() - .enumerate() - .flat_map(|(i, st)| { - st.params - .as_object() - .into_iter() - .flatten() - .map(move |(k, v)| json!({"step": i + 1, "tool": st.tool, "name": k, "default": v})) - }) - .collect(); - let tool = json!({"format": "RustGIS web tool", "version": 1, "name": m.name, "steps": m.steps, "parameters": params}); - Ok((m.name.clone(), serde_json::to_vec_pretty(&tool).map_err(|e| e.to_string())?, m.name.clone())) + let name = m.name.clone(); + let (bytes, report) = crate::durable::output(m, None, |m| { + let params: Vec = m + .steps + .iter() + .enumerate() + .flat_map(|(i, st)| { + st.params + .as_object() + .into_iter() + .flatten() + .map(move |(k, v)| json!({"step": i + 1, "tool": st.tool, "name": k, "default": v})) + }) + .collect(); + let tool = json!({"format": "RustGIS web tool", "version": 1, "name": m.name, "steps": m.steps, "parameters": params}); + serde_json::to_vec_pretty(&tool).map_err(|e| e.to_string()) + })?; + crate::durable::notify(s, &report); + Ok((name.clone(), bytes, name)) } "webScene" => { let mut m = map_of(s, p)?; diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index f1d0d33aa..d659d09a5 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -522,7 +522,9 @@ fn save_model(s: &mut Session, p: &Value) -> CmdResult { /// A model as a `*.model.json` file: written to `path`, or its text returned. fn export_model(s: &mut Session, p: &Value) -> CmdResult { let (name, g, _) = model_graph(s, p)?; - let text = serde_json::to_string_pretty(&crate::model::to_file(&name, &g)).map_err(|e| e.to_string())?; + let (text, report) = + crate::durable::output_owned(g, |g| serde_json::to_string_pretty(&crate::model::to_file(&name, g)).map_err(|e| e.to_string()))?; + crate::durable::notify(s, &report); let file: String = name.chars().map(|c| if c.is_alphanumeric() || c == '-' || c == '_' { c } else { '_' }).collect(); match ps(p, "path") { Some(path) => { @@ -630,21 +632,10 @@ pub fn refused_in_script(id: &str) -> bool { pub fn script_commands(script: &str) -> Result, String> { let mut out = Vec::new(); for (n, line) in script.lines().enumerate().take(MAX_SCRIPT_LINES) { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let (id, rest) = line.split_once(char::is_whitespace).unwrap_or((line, "")); - let id = id.trim_end_matches('='); - let params: Value = if rest.trim().is_empty() { - Value::Null - } else { - serde_json::from_str(rest.trim()).map_err(|e| format!("line {}: {e}", n + 1))? - }; - if refused_in_script(id) { - return Err(format!("line {}: '{id}' can't run in a script", n + 1)); + if let Some(parsed) = crate::script_line(line) { + let (id, params) = parsed.map_err(|e| format!("line {}: {e}", n + 1))?; + out.push((n + 1, id, params)); } - out.push((n + 1, id.to_string(), params)); } Ok(out) } diff --git a/crates/engine/src/durable.rs b/crates/engine/src/durable.rs new file mode 100644 index 000000000..635ff2973 --- /dev/null +++ b/crates/engine/src/durable.rs @@ -0,0 +1,700 @@ +//! Controlled, synchronous durable output. Preparation owns every replacement before the first +//! mutation. The edit list is also the rollback journal; applying and undoing it only swaps +//! disjoint whole fields at fixed locations. No observer, policy planner or callback may see the +//! temporary projection. Formatters are internal, borrow only the projection and never reenter. + +use std::collections::{HashSet, hash_map::RandomState}; +use std::hash::BuildHasher; +use std::sync::{Mutex, OnceLock}; + +use serde_json::Value; + +/// Counted losses at the output boundary. Generated display text is cleaned without a notice. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Report { + pub(crate) addresses: u32, + pub(crate) videos: u32, + pub(crate) steps: u32, + pub(crate) lines: u32, + pub(crate) withheld: u32, + pub(crate) unchecked: u32, + pub(crate) not_remembered: bool, + pub(crate) lost: u32, + pub(crate) not_retained: u32, +} + +impl Report { + /// The existing `project.save` result's `stripped` count: changed layer addresses only. + pub fn layer_addresses(&self) -> u32 { + self.addresses + } + + /// Addresses whose keys the receiving session lost during peer replacement. + pub fn lost(&self) -> u32 { + self.lost + } + + /// One notice for each kind of loss, and none for a clean output. + pub fn notices(&self) -> Vec { + let mut out = Vec::new(); + for (n, text) in [ + ( + self.addresses, + "layer address(es) were saved without their keys, tokens or passwords: after RustGIS closes, add those layers again to give them", + ), + (self.videos, "video address(es) were written without their keys, tokens or passwords"), + (self.steps, "executable step(s) were written without their credentials; supply them again before replay"), + (self.lines, "command line(s) were written without their credentials; supply them again before replay"), + (self.withheld, "credential-bearing slot(s) could not be read and were withheld"), + (self.unchecked, "step(s) contain opaque parameters that may hold credentials and were kept unchecked"), + (self.not_retained, "address(es) exceeded the bridge retention limit; their keys will not survive a round trip"), + (self.lost, "layer address(es) lost their keys when a peer sent back its project; add them again in the receiving session"), + ] { + if n > 0 { + out.push(format!("{n} {text}")); + } + } + if self.not_remembered { + out.push("This output's credential restoration exceeds the memory limit; reopening it will not bring its keys back".into()); + } + out + } + + /// Adds the independently prepared losses of a nested typed document. + pub(crate) fn merge(&mut self, other: &Self) { + self.addresses = self.addresses.saturating_add(other.addresses); + self.videos = self.videos.saturating_add(other.videos); + self.steps = self.steps.saturating_add(other.steps); + self.lines = self.lines.saturating_add(other.lines); + self.withheld = self.withheld.saturating_add(other.withheld); + self.unchecked = self.unchecked.saturating_add(other.unchecked); + self.not_remembered |= other.not_remembered; + self.lost = self.lost.saturating_add(other.lost); + self.not_retained = self.not_retained.saturating_add(other.not_retained); + } +} + +/// A child owner in a typed location, independent of any field's current contents. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) enum Owner { + Map(usize), + Area(usize), + Table(usize), + Task(usize), + Model(usize), + History(usize), + Notebook(usize), + Presentation(usize), + Graph, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) enum Leaf { + Map(rustgis_map::SlotPos), + Layer(rustgis_map::LayerRole), + Table, + Task(usize), + ModelStep(usize), + ModelNode(usize), + HistoryParams, + HistoryCommand, + Message(usize), + CellSource(usize), + CellOutput(usize), + Video(usize), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) struct At { + pub(crate) owners: Vec, + pub(crate) leaf: Leaf, +} + +#[derive(Clone, Debug, PartialEq)] +enum Payload { + Text(String), + Json(Value), +} + +pub(crate) enum FieldMut<'a> { + Text(&'a mut String), + Json(&'a mut Value), +} + +impl FieldMut<'_> { + /// Type-checks before swapping; after the swap the caller immediately advances its journal. + fn swap(self, held: &mut Payload) -> bool { + match (self, held) { + (Self::Text(field), Payload::Text(value)) => std::mem::swap(field, value), + (Self::Json(field), Payload::Json(value)) => std::mem::swap(field, value), + _ => return false, + } + true + } + + /// Clones only one changed owning field, after the live projection has been rolled back. + fn payload(&self) -> Payload { + match self { + Self::Text(v) => Payload::Text((**v).clone()), + Self::Json(v) => Payload::Json((**v).clone()), + } + } + + /// Compares the actual parsed field to the recorded written value without policy dispatch. + fn equals(&self, value: &Payload) -> bool { + match (self, value) { + (Self::Text(a), Payload::Text(b)) => **a == *b, + (Self::Json(a), Payload::Json(b)) => **a == *b, + _ => false, + } + } + + /// Counts retained text/JSON bytes before copying into the bounded reopen memory. + fn bytes(&self) -> usize { + match self { + Self::Text(v) => v.len(), + Self::Json(v) => json_bytes(v), + } + } +} + +/// Counts payload without allocating a serialized copy, including JSON punctuation/escaping. +fn json_bytes(v: &Value) -> usize { + struct Counter(usize); + impl std::io::Write for Counter { + /// Counts the serializer's bytes with saturating arithmetic, retaining no payload. + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.0 = self.0.saturating_add(b.len()); + Ok(b.len()) + } + /// The counting writer has no buffered bytes to flush. + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut c = Counter(0); + if serde_json::to_writer(&mut c, v).is_err() { + return usize::MAX; + } + c.0 +} + +pub(crate) trait Durable { + /// Preparation is read-only, exhaustive over owning fields, and offers each leaf once. + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>); + /// Resolves recorded indices only, through plain owned fields, without allocation or policy. + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option>; +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + Credential, + Path, + Generated, +} +struct Edit { + at: At, + held: Payload, + kind: Kind, +} + +pub(crate) struct Prepare<'a> { + owners: Vec, + edits: Vec, + pub(crate) report: Report, + relative_to: Option<&'a str>, + pub(crate) depth: usize, +} + +impl<'a> Prepare<'a> { + /// Makes every replacement and journal allocation before the live borrow is mutated. + fn new(relative_to: Option<&'a str>, depth: usize) -> Self { + Self { owners: Vec::new(), edits: Vec::new(), report: Report::default(), relative_to, depth } + } + + /// Prepares one child under a fixed, typed owning location. + pub(crate) fn child(&mut self, owner: Owner, value: &impl Durable) { + self.owners.push(owner); + value.prepare(self); + self.owners.pop(); + } + + /// Records a single replacement of a whole owning field, never a subfield of an edit. + fn edit(&mut self, leaf: Leaf, held: Payload, kind: Kind) { + self.edits.push(Edit { at: At { owners: self.owners.clone(), leaf }, held, kind }); + } + + /// Cleans an address and then makes its local path relative, in one journal entry. + pub(crate) fn string(&mut self, leaf: Leaf, value: &str, address: bool, local_path: bool) { + let mut replacement = None; + let mut kind = Kind::Path; + if address { + replacement = match public_address(value) { + Kept::Unchanged => None, + Kept::Changed(s) => Some(s), + Kept::Withheld(prefix) if prefix != value => { + self.report.withheld = self.report.withheld.saturating_add(1); + Some(prefix.into()) + } + Kept::Withheld(_) => None, + }; + if replacement.is_some() { + kind = Kind::Credential; + if matches!(leaf, Leaf::Video(_)) { + self.report.videos = self.report.videos.saturating_add(1); + } else { + self.report.addresses = self.report.addresses.saturating_add(1); + } + } + } + let current = replacement.as_deref().unwrap_or(value); + if local_path + && let Some(dir) = self.relative_to + && rustgis_map::paths::is_absolute_local(current) + && let Some(relative) = rustgis_map::paths::relative_to(dir, current).filter(|r| rustgis_map::paths::is_local(r)) + { + replacement = Some(relative); + } + if let Some(s) = replacement.filter(|s| s != value) { + self.edit(leaf, Payload::Text(s), kind); + } + } + + /// Prepares a command's replayable parameters using explicit data/configuration roles. + pub(crate) fn command(&mut self, leaf: Leaf, id: &str, value: &Value) { + let (clean, report) = crate::roles::command(id, value, self.depth); + self.report.merge(&report); + if let Some(clean) = clean { + self.report.steps = self.report.steps.saturating_add(1); + self.edit(leaf, Payload::Json(clean), Kind::Credential); + } + } + + /// Model and history tool params have the same roles as the corresponding `gp.run` params. + pub(crate) fn tool(&mut self, leaf: Leaf, id: &str, value: &Value) { + let (clean, report) = crate::roles::tool(id, value, self.depth); + self.report.merge(&report); + if let Some(clean) = clean { + self.report.steps = self.report.steps.saturating_add(1); + self.edit(leaf, Payload::Json(clean), Kind::Credential); + } + } + + /// Rewrites only changed command lines; comments, data and foreign-language cells stay exact. + pub(crate) fn script(&mut self, leaf: Leaf, source: &str) { + let (clean, report) = crate::roles::script(source, self.depth); + self.report.merge(&report); + if let Some(clean) = clean { + self.edit(leaf, Payload::Text(clean), Kind::Credential); + } + } + + /// Cleans only addresses in text RustGIS generated, without claiming opaque text detection. + pub(crate) fn generated(&mut self, leaf: Leaf, source: &str) { + let (clean, n) = rustgis_data::address::scrub_addresses(source); + if n > 0 { + self.edit(leaf, Payload::Text(clean.into_owned()), Kind::Generated); + } + } +} + +struct Transaction<'a, T: Durable> { + live: &'a mut T, + edits: Vec, + applied: usize, + defect: bool, +} + +impl Transaction<'_, T> { + /// Guard is already armed. The original becomes journal-owned at each infallible swap. + fn apply(&mut self) -> Result<(), String> { + for edit in &mut self.edits { + let field = self.live.field_mut(&edit.at.owners, &edit.at.leaf).ok_or("internal output location changed")?; + if !field.swap(&mut edit.held) { + return Err("internal output field type changed".into()); + } + self.applied += 1; + } + Ok(()) + } + + /// Restores the applied prefix in reverse, with no discovery, parsing or allocation on the + /// valid path. A resolver defect is an internal error, never a successful partial output. + fn rollback(&mut self) { + while self.applied > 0 { + self.applied -= 1; + if let Some(edit) = self.edits.get_mut(self.applied) { + match self.live.field_mut(&edit.at.owners, &edit.at.leaf) { + Some(field) => self.defect |= !field.swap(&mut edit.held), + None => self.defect = true, + } + } else { + self.defect = true; + } + } + } +} + +impl Drop for Transaction<'_, T> { + /// Restores the applied journal before releasing the live borrow, including native unwinding. + fn drop(&mut self) { + self.rollback(); + } +} + +/// Rejects repeated locations before arming, so one edit can never overwrite another's original. +fn prepared<'a>(value: &impl Durable, relative_to: Option<&'a str>, depth: usize) -> Result, String> { + let mut p = Prepare::new(relative_to, depth); + value.prepare(&mut p); + let mut seen = HashSet::new(); + if p.edits.iter().any(|e| !seen.insert(&e.at)) { + return Err("internal duplicate output location".into()); + } + drop(seen); + Ok(p) +} + +/// No whole-project clone: every live field is restored on success, error and native unwind. +/// Abort/OOM/process death cannot run Drop; no temporarily edited state is persisted by them. +pub(crate) fn output( + value: &mut T, + relative_to: Option<&str>, + f: impl FnOnce(&T) -> Result, +) -> Result<(R, Report), String> { + let p = prepared(value, relative_to, 0)?; + let mut t = Transaction { live: value, edits: p.edits, applied: 0, defect: false }; + t.apply()?; + let result = f(t.live); + t.rollback(); + if t.defect { + return Err("internal output rollback failed".into()); + } + result.map(|r| (r, p.report)) +} + +/// Consumes a draft the writer already owns, after structural edits; no extra project clone. +pub(crate) fn output_owned(value: T, f: impl FnOnce(&T) -> Result) -> Result<(R, Report), String> { + output_owned_at(value, 0, f) +} + +/// Nested documents share the executable nesting budget, instead of starting a fresh one. +pub(crate) fn output_owned_at( + value: T, + depth: usize, + f: impl FnOnce(&T) -> Result, +) -> Result<(R, Report), String> { + output_owned_changed(value, depth, |value, _| f(value)) +} + +/// Typed documents serialize only when an actual field changed, independently of notices. +pub(crate) fn output_owned_changed( + mut value: T, + depth: usize, + f: impl FnOnce(&T, bool) -> Result, +) -> Result<(R, Report), String> { + let p = prepared(&value, None, depth)?; + let changed = !p.edits.is_empty(); + for mut edit in p.edits { + if !value.field_mut(&edit.at.owners, &edit.at.leaf).ok_or("internal output location changed")?.swap(&mut edit.held) { + return Err("internal output field type changed".into()); + } + } + f(&value, changed).map(|r| (r, p.report)) +} + +struct Remembered { + at: At, + written: Payload, + original: Payload, +} +struct Written { + len: usize, + hash: u64, + bytes: usize, + edits: Vec, +} +const MAX_ENTRY: usize = 1 << 20; +const MAX_TOTAL: usize = 8 << 20; +const MAX_ENTRIES: usize = 32; + +/// Process-local identity, never a public hash or a peer's authority. +fn identity(text: &str) -> (usize, u64) { + static KEY: OnceLock = OnceLock::new(); + (text.len(), KEY.get_or_init(RandomState::new).hash_one(text.as_bytes())) +} + +/// The explicit-save cache is independent of the output rollback journal. +fn kept() -> std::sync::MutexGuard<'static, Vec> { + static KEPT: OnceLock>> = OnceLock::new(); + KEPT.get_or_init(Default::default).lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Remembers only credential edits after rollback, with exact written-value equality on reopen. +pub(crate) fn output_remembered( + value: &mut T, + dir: Option<&str>, + f: impl FnOnce(&T) -> Result, +) -> Result<(String, Report), String> { + let p = prepared(value, dir, 0)?; + let mut t = Transaction { live: value, edits: p.edits, applied: 0, defect: false }; + t.apply()?; + let result = f(t.live); + t.rollback(); + if t.defect { + return Err("internal output rollback failed".into()); + } + let text = result?; + let mut report = p.report; + let (len, hash) = identity(&text); + let mut bytes = 0usize; + for e in t.edits.iter().filter(|e| e.kind == Kind::Credential) { + let original = t.live.field_mut(&e.at.owners, &e.at.leaf).ok_or("internal output location changed")?; + let written = match &e.held { + Payload::Text(v) => v.len(), + Payload::Json(v) => json_bytes(v), + }; + bytes = bytes + .saturating_add(original.bytes()) + .saturating_add(written) + .saturating_add(std::mem::size_of::()) + .saturating_add(e.at.owners.len().saturating_mul(std::mem::size_of::())); + } + let mut cache = kept(); + cache.retain(|e| (e.len, e.hash) != (len, hash)); + if bytes > MAX_ENTRY { + report.not_remembered = true; + } else if bytes > 0 { + let mut edits = Vec::new(); + for e in std::mem::take(&mut t.edits).into_iter().filter(|e| e.kind == Kind::Credential) { + let original = t.live.field_mut(&e.at.owners, &e.at.leaf).ok_or("internal output location changed")?.payload(); + edits.push(Remembered { at: e.at, written: e.held, original }); + } + insert(&mut cache, Written { len, hash, bytes, edits }); + } + Ok((text, report)) +} + +/// Oldest explicit saves go first, bounded by both count and total retained payload. +fn insert(cache: &mut Vec, entry: Written) { + while cache.len() >= MAX_ENTRIES || cache.iter().fold(entry.bytes, |n, e| n.saturating_add(e.bytes)) > MAX_TOTAL { + cache.remove(0); + } + cache.push(entry); +} + +/// Runs before normalize/path resolution. Imports and peer ingress never call this function. +pub(crate) fn restore(value: &mut impl Durable, text: &str) { + let id = identity(text); + let cache = kept(); + if let Some(entry) = cache.iter().find(|e| (e.len, e.hash) == id) { + for e in &entry.edits { + if let Some(field) = value.field_mut(&e.at.owners, &e.at.leaf) + && field.equals(&e.written) + { + let mut original = e.original.clone(); + let _ = field.swap(&mut original); + } + } + } +} + +pub(crate) enum Kept { + Unchanged, + Changed(String), + Withheld(&'static str), +} + +/// Prefix recognition precedes parsing: malformed recognized wrappers never become opaque. +pub(crate) fn public_address(address: &str) -> Kept { + public_address_at(address, 0) +} + +/// Wrapper recursion is bounded independently of executable document nesting. +fn public_address_at(address: &str, depth: usize) -> Kept { + use rustgis_data::{ + address::{Public, public_web_address}, + s3_address, + }; + if let Some((public, _)) = rustgis_data::source::myconn::split_password(address) { + return if public == address { Kept::Unchanged } else { Kept::Changed(public) }; + } + if let Some((public, _)) = rustgis_data::source::msconn::split_password(address) { + return if public == address { Kept::Unchanged } else { Kept::Changed(public) }; + } + let text = address.trim_start(); + let lower = text.to_ascii_lowercase(); + let recognized = ["rustgis-anonymous:", "rustgis-s3:", "pmtiles:", "cog:", "postgresql:", "postgres:", "pg:", "https:", "http:", "s3:"] + .into_iter() + .find(|p| lower.starts_with(p)); + let Some(prefix) = recognized else { return Kept::Unchanged }; + if depth >= 8 { + return Kept::Withheld(prefix); + } + if prefix == "rustgis-anonymous:" { + let Some(inner) = text.get(prefix.len()..) else { return Kept::Withheld(prefix) }; + return match public_address_at(inner, depth + 1) { + Kept::Unchanged => Kept::Unchanged, + Kept::Changed(s) => Kept::Changed(format!("{prefix}{s}")), + Kept::Withheld(_) => Kept::Withheld(prefix), + }; + } + if matches!(prefix, "pmtiles:" | "cog:") { + let Some((head, archive)) = crate::sources::template_archive(text) else { return Kept::Withheld(prefix) }; + let file = crate::render::rustgis_tiles::cog::split(archive).0; + let look = archive.get(file.len()..).unwrap_or(""); + return match public_address_at(file, depth + 1) { + Kept::Unchanged => Kept::Unchanged, + Kept::Changed(s) => Kept::Changed(format!("{head}{s}{look}")), + Kept::Withheld(_) => Kept::Withheld(prefix), + }; + } + if prefix == "rustgis-s3:" { + let file = crate::render::rustgis_tiles::cog::split(text).0; + let look = text.get(file.len()..).unwrap_or(""); + let Ok(Some(b)) = s3_address::parse(file) else { return Kept::Withheld(prefix) }; + return match public_address_at(b.url, depth + 1) { + Kept::Unchanged => Kept::Unchanged, + Kept::Changed(s) => match s3_address::bind(b.binding, b.endpoint, &s) { + Ok(s) => Kept::Changed(format!("{s}{look}")), + Err(_) => Kept::Withheld(prefix), + }, + Kept::Withheld(_) => Kept::Withheld(prefix), + }; + } + if matches!(prefix, "postgresql:" | "postgres:" | "pg:") { + return match rustgis_data::source::pgconn::public_label(text) { + Some(s) if s == address => Kept::Unchanged, + Some(s) => Kept::Changed(s), + None => Kept::Withheld(prefix), + }; + } + match public_web_address(address) { + Public::Unchanged => Kept::Unchanged, + Public::Changed { address, .. } => Kept::Changed(address), + Public::Withheld => Kept::Withheld(prefix), + } +} + +/// Emits the report only after rollback, when the session is whole again. +pub(crate) fn notify(s: &mut crate::Session, report: &Report) { + for text in report.notices() { + s.notify(text, false); + } +} + +impl Durable for Vec { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + for (i, v) in self.iter().enumerate() { + p.child(Owner::History(i), v); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + let (Owner::History(i), rest) = owners.split_first()? else { return None }; + self.get_mut(*i)?.field_mut(rest, leaf) + } +} + +impl Durable for rustgis_map::Map { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + for s in self.slots() { + p.string(Leaf::Map(s.at), s.value, s.address, s.local_path); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + if !owners.is_empty() { + return None; + } + let Leaf::Map(at) = leaf else { return None }; + self.slot_mut(at).map(FieldMut::Text) + } +} + +impl Durable for rustgis_map::Layer { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + for (role, value, address, path) in self.output_slots() { + p.string(Leaf::Layer(role), value, address, path); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + if !owners.is_empty() { + return None; + } + let Leaf::Layer(at) = leaf else { return None }; + self.output_slot_mut(*at).map(FieldMut::Text) + } +} + +impl Durable for rustgis_map::presentation::Presentation { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + use rustgis_map::presentation::{Page, PageKind}; + let Self { name: _, width: _, height: _, pages } = self; + for (i, Page { title: _, kind, notes: _, seconds: _ }) in pages.iter().enumerate() { + match kind { + PageKind::Video { source, poster: _, caption: _ } => p.string(Leaf::Video(i), source, true, false), + PageKind::Map { map: _, center: _, width_m: _ } + | PageKind::Blank { text: _, background: _ } + | PageKind::Image { image: _, caption: _ } => {} + } + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + if !owners.is_empty() { + return None; + } + let Leaf::Video(i) = leaf else { return None }; + match &mut self.pages.get_mut(*i)?.kind { + rustgis_map::presentation::PageKind::Video { source, .. } => Some(FieldMut::Text(source)), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn restore_memory_has_entry_total_and_count_bounds() { + let marker = "RGSYNTH_1"; + let mut p = crate::Project::default(); + p.tasks.push(crate::project::Task { + name: "large".into(), + description: String::new(), + steps: vec![crate::project::TaskStep { + title: String::new(), + instructions: String::new(), + command: "layer.addUrl".into(), + params: serde_json::json!({"token": marker.repeat(MAX_ENTRY / marker.len() + 1)}), + }], + }); + let (text, report) = p.saved_text(None).unwrap(); + assert!(report.not_remembered && report.notices().iter().any(|n| n.contains("memory limit"))); + assert!(crate::Project::from_json(&text).unwrap().tasks[0].steps[0].params["token"].as_str().unwrap().is_empty()); + let mut cache = Vec::new(); + for hash in 0..40 { + insert(&mut cache, Written { len: 0, hash, bytes: MAX_ENTRY, edits: Vec::new() }); + } + assert_eq!(cache.len(), MAX_TOTAL / MAX_ENTRY); + assert_eq!(cache.first().unwrap().hash, 32); + let mut cache = Vec::new(); + for hash in 0..40 { + insert(&mut cache, Written { len: 0, hash, bytes: 1, edits: Vec::new() }); + } + assert_eq!(cache.len(), MAX_ENTRIES); + assert_eq!(cache.first().unwrap().hash, 8); + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index f26ecc228..673da4aed 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -7,12 +7,19 @@ pub mod cmd; pub mod connections; +pub mod durable; pub mod geometry_calc; pub mod model; +pub mod peer; #[cfg(not(target_arch = "wasm32"))] pub mod prefs; pub mod project; +mod roles; +pub use peer::Peer; +mod parameters; pub mod project_log; +pub use parameters::{params_envelope, script_line}; +pub use project_log::{LogCopy, log_params}; pub mod s3_connections; pub mod s3_keys; #[cfg(test)] @@ -432,6 +439,18 @@ pub struct Notice { pub error: bool, } +/// One counted, already-redacted loss, independent of the changes a command can roll back. +struct LossNotice { + template: &'static str, + count: u32, + text: rustgis_data::diagnostics::Redacted, + source: Option<&'static str>, +} + +/// At most 256 KiB of redacted text per outer command (32 kinds, redaction's 8 KiB bound). +/// Further losses are counted in one overflow notice rather than silently discarded. +const MAX_LOSS_NOTICE_KINDS: usize = 32; + const UNDO_LIMIT: usize = 40; struct ProjectFiles { @@ -551,6 +570,8 @@ pub struct Session { /// the command came (the app, an agent, a script): the interface drops what it kept for the /// old one. pub opened: u64, + /// Session-local last bridge exchange; absent from project files and undo snapshots. + pub(crate) exchange: Option, pub tool: Tool, /// The layer selected in the Layers pane (the one the Style and Data tabs act on). pub active_layer: Option, @@ -567,6 +588,10 @@ pub struct Session { /// camera then: moved since, the map stays. pub style_view: Option<(String, Camera)>, pub notices: Vec, + /// Counted losses await the outermost guard's cleanup, including failed nested commands. + /// Runtime-only: never in Project, undo or PutAside, so replacement cannot erase them. + loss_notices: Vec, + loss_notice_overflow: u32, /// Troubleshooting history, independent of the active pane, project files and undo. pub diagnostics: rustgis_data::diagnostics::History, /// Notices given in this session, including ones no longer kept (see [`Session::notices_since`]), @@ -876,6 +901,7 @@ impl Session { registry: Registry { commands: cmd::all() }, revision: 1, opened: 0, + exchange: None, tool: Tool::Explore, active_layer: None, layer_selection: Vec::new(), @@ -883,6 +909,8 @@ impl Session { view_size: (1200.0, 720.0), style_view: None, notices: Vec::new(), + loss_notices: Vec::new(), + loss_notice_overflow: 0, diagnostics: rustgis_data::diagnostics::History::default(), notices_given: 0, table_layer: None, @@ -1351,7 +1379,7 @@ impl Session { .skip(history_from) .filter(|record| !record.ok && rustgis_analysis::tool(&record.id).is_some_and(cmd::gp::writes_files)) { - self.project.gp_history.push(record); + self.push_gp_record(record); recorded_failure = true; } if recorded_failure { @@ -1380,6 +1408,11 @@ impl Session { } self.fix_active_layer(); self.revision = self.revision.wrapping_add(1); + // Nested guards leave their losses pending: only after the outermost rollback has + // withdrawn completion notices may these go to ordinary notice consumers. + if self.depth == 0 { + self.flush_loss_notices(); + } res } @@ -1545,8 +1578,61 @@ impl Session { self.notice_redacted(text, if error { Severity::Error } else { Severity::Info }); } + /// A counted loss that remains true after rollback, unlike a completion such as "Deleted + /// 3 features". Only static engine templates with `{n}` and a count enter this channel. + /// Counts saturate and repeated templates aggregate once per outer command. Text is redacted + /// before retention; at most 32 kinds are held, with excess counts reported as one overflow. + /// Outside a guard it is published immediately. A native panic caught by `guarded` follows + /// the same outermost drain as an error; nothing is saved in project or undo state. + pub(crate) fn notify_loss(&mut self, template: &'static str, count: u32) { + use rustgis_data::diagnostics::{Redacted, Severity}; + if count == 0 { + return; + } + if self.depth == 0 { + self.notice_redacted(Redacted::new(&template.replace("{n}", &count.to_string())), Severity::Info); + } else if let Some(loss) = self.loss_notices.iter_mut().find(|loss| loss.template == template) { + loss.count = loss.count.saturating_add(count); + loss.text = Redacted::new(&template.replace("{n}", &loss.count.to_string())); + } else if self.loss_notices.len() < MAX_LOSS_NOTICE_KINDS { + self.loss_notices.push(LossNotice { + template, + count, + text: Redacted::new(&template.replace("{n}", &count.to_string())), + source: self.diagnostic_source, + }); + } else { + self.loss_notice_overflow = self.loss_notice_overflow.saturating_add(count); + } + } + + /// Publishes and empties runtime losses only after outermost success/error/panic cleanup. + /// Stored attribution keeps nested command losses assigned to their original command. + fn flush_loss_notices(&mut self) { + use rustgis_data::diagnostics::Severity; + for loss in std::mem::take(&mut self.loss_notices) { + self.notice_redacted_from(loss.text, Severity::Info, loss.source); + } + let overflow = std::mem::take(&mut self.loss_notice_overflow); + if overflow > 0 { + self.notice(format!("{overflow} further loss(es) could not be described"), Severity::Info); + } + } + + /// Uses current attribution for ordinary notices; delayed losses retain their own source. fn notice_redacted(&mut self, text: rustgis_data::diagnostics::Redacted, severity: rustgis_data::diagnostics::Severity) { - self.diagnostics.push_redacted(severity, self.diagnostic_source.unwrap_or("Application"), &text); + self.notice_redacted_from(text, severity, self.diagnostic_source); + } + + /// The one publication path for the ordinary notice queue and diagnostics; called once per + /// loss at drain, so neither sink sees a duplicate from queuing and publication. + fn notice_redacted_from( + &mut self, + text: rustgis_data::diagnostics::Redacted, + severity: rustgis_data::diagnostics::Severity, + source: Option<&'static str>, + ) { + self.diagnostics.push_redacted(severity, source.unwrap_or("Application"), &text); self.notices.push(Notice { text: text.into_string(), error: severity == rustgis_data::diagnostics::Severity::Error }); self.notices_given += 1; if self.notices.len() > 200 { @@ -1962,3 +2048,40 @@ impl Session { self.project.name.clone() } } + +impl durable::Durable for GpRecord { + fn prepare(&self, p: &mut durable::Prepare<'_>) { + use durable::Leaf; + let Self { tool, id, params, command, at: _, ok: _, messages, millis: _, outputs: _ } = self; + let tool = if id.is_empty() { cmd::gp::tool_id(tool).unwrap_or("") } else { id }; + p.tool(Leaf::HistoryParams, tool, params); + if let Some((id, params)) = command { + p.command(Leaf::HistoryCommand, id, params); + } + for (i, message) in messages.iter().enumerate() { + p.generated(Leaf::Message(i), message); + } + } + fn field_mut(&mut self, owners: &[durable::Owner], leaf: &durable::Leaf) -> Option> { + use durable::{FieldMut, Leaf}; + if !owners.is_empty() { + return None; + } + match leaf { + Leaf::HistoryParams => Some(FieldMut::Json(&mut self.params)), + Leaf::HistoryCommand => Some(FieldMut::Json(&mut self.command.as_mut()?.1)), + Leaf::Message(i) => self.messages.get_mut(*i).map(FieldMut::Text), + _ => None, + } + } +} + +impl Session { + /// The single history production seam: retained generated messages never keep address keys. + pub(crate) fn push_gp_record(&mut self, mut record: GpRecord) { + for message in &mut record.messages { + *message = rustgis_data::address::scrub_addresses(message).0.into_owned(); + } + self.project.gp_history.push(record); + } +} diff --git a/crates/engine/src/parameters.rs b/crates/engine/src/parameters.rs new file mode 100644 index 000000000..410aff8c5 --- /dev/null +++ b/crates/engine/src/parameters.rs @@ -0,0 +1,141 @@ +//! Authoritative adapter envelope and script parsing. Redacted log/display copies never enter +//! this module: callers retain the returned private value for planning and execution. + +use serde_json::{Value, json}; + +/// The parameters an adapter executes: an object or JSON text of one. Missing, null and blank +/// mean `{}`; other input fails, so a malformed request cannot act with unintended defaults. +pub fn params_envelope(params: Option<&Value>) -> Result { + match params { + None | Some(Value::Null) => Ok(json!({})), + Some(v @ Value::Object(_)) => Ok(v.clone()), + Some(Value::String(t)) if t.trim().is_empty() => Ok(json!({})), + Some(Value::String(t)) => match serde_json::from_str::(t) { + Ok(v @ Value::Object(_)) => Ok(v), + Ok(_) => Err("params must be a JSON object".into()), + Err(e) => Err(format!("params is not valid JSON: {e}")), + }, + Some(_) => Err("params must be a JSON object".into()), + } +} + +/// One executable script line, preserving the executor's `id[=] {json}` grammar. Unlike an +/// adapter envelope, a script's JSON value is passed as written; no parameters means null. +/// Blank and comment lines have no command. Errors are annotated with line numbers by callers. +pub fn script_line(line: &str) -> Option> { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + let (id, rest) = line.split_once(char::is_whitespace).unwrap_or((line, "")); + let id = id.trim_end_matches('='); + let params = if rest.trim().is_empty() { + Value::Null + } else { + match serde_json::from_str(rest.trim()) { + Ok(value) => value, + Err(e) => return Some(Err(e.to_string())), + } + }; + if crate::cmd::workflows::refused_in_script(id) { + return Some(Err(format!("'{id}' can't run in a script"))); + } + Some(Ok((id.to_string(), params))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn envelopes_and_script_lines_keep_the_executor_grammar() { + let object = json!({"layer": "private source", "token": "RGSYNTH_params_20261103"}); + assert_eq!(params_envelope(Some(&object)), params_envelope(Some(&json!(object.to_string())))); + for value in [json!(42), json!([]), json!("[1]"), json!("false"), json!("{bad")] { + assert!(params_envelope(Some(&value)).is_err(), "{value}"); + } + for value in [Value::Null, json!(" \t")] { + assert_eq!(params_envelope(Some(&value)).unwrap(), json!({})); + } + assert_eq!(params_envelope(None).unwrap(), json!({})); + assert!(script_line(" # comment").is_none()); + assert_eq!(script_line(" map.zoomIn= {} ").unwrap().unwrap(), ("map.zoomIn".into(), json!({}))); + assert_eq!(script_line("map.zoomIn").unwrap().unwrap().1, Value::Null); + assert_eq!(script_line("map.zoomIn []").unwrap().unwrap().1, json!([])); + assert!(script_line("map.zoomIn {bad").unwrap().is_err()); + assert!(script_line("script.run {}").unwrap().is_err()); + } + + /// Supplied documents and an explicitly supplied final template keep this command test + /// network-free while exercising the real Session notice boundary. + #[test] + fn service_discovery_reports_withheld_destinations_and_clean_input_is_silent() { + for (path, count) in [("tiles", 0), ("%252e/tiles", 1)] { + let href = format!("https://h.invalid/svc/{path}?token=RGSYNTH_notice_20261103"); + let document = json!({"title": "roads", "links": [{"rel": "tilesets-map", "href": href}]}).to_string(); + let mut params = json!({"url": "https://h.invalid/svc", "type": "ogcapi", "text": document}); + let mut session = crate::Session::new(crate::Project::default()); + session.run("service.layers", ¶ms).unwrap(); + let notices = std::mem::take(&mut session.notices); + assert_eq!(notices.len(), count); + assert!(notices.iter().all(|n| n.text.contains("1 address(es)") && !n.text.contains("RGSYNTH_"))); + params["layer"] = json!("map"); + params["tiles"] = json!("https://h.invalid/explicit/{z}/{y}/{x}"); + session.run("service.add", ¶ms).unwrap(); + let notices = std::mem::take(&mut session.notices); + assert_eq!(notices.len(), count); + assert!(notices.iter().all(|n| n.text.contains("1 address(es)") && !n.error)); + // All fail after discovery counted its withheld destinations. The forwarding notice + // is independent of the unchanged command error, and is emitted once on either path. + for (changes, expected) in [ + (json!({"time": {}}), "roads (map tiles): time is a date, \"slider\" or \"default\""), + (json!({"tiles": "not-a-template"}), "not-a-template: not a tile template"), + (json!({"layer": "missing"}), "https://h.invalid/svc: no layer 'missing' (service.layers lists them)"), + ] { + let mut bad = params.clone(); + for (key, value) in changes.as_object().unwrap() { + bad[key] = value.clone(); + } + assert_eq!(session.run("service.add", &bad).unwrap_err(), expected); + let notices = std::mem::take(&mut session.notices); + let forwarding = notices.iter().filter(|n| !n.error && n.text.contains("address(es) the service listed")).count(); + assert_eq!(forwarding, count); + assert_eq!(notices.iter().filter(|n| n.error).count(), 1, "the original error is still reported"); + assert_eq!(notices.len(), count + 1, "one aggregated forwarding notice, clean errors silent about forwarding"); + } + } + } + + /// Nested success and a later caught panic must retain one counted loss, while the + /// completion that was rolled back disappears. The next guard must publish nothing stale. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn nested_loss_notices_drain_once_after_panic_cleanup() { + let mut session = crate::Session::new(crate::Project::default()); + let name = session.project.name.clone(); + let mark = session.notice_mark(); + const LOSS: &str = "{n} resource(s) were omitted at https://h.invalid/x?token=RGSYNTH_2"; + let result = session.guarded("Loss rollback", true, |s| { + s.guarded("Nested loss", true, |s| { + s.notify_loss(LOSS, 1); + Ok(Value::Null) + })?; + assert!(s.notices_since(mark).is_empty(), "the outer guard has not finished"); + s.notify_loss(LOSS, 2); + s.project.name = "temporary".into(); + s.notify("Deleted 3 features", false); + panic!("synthetic loss rollback"); + }); + assert_eq!(result.unwrap_err(), "Loss rollback failed unexpectedly; the project was restored"); + assert_eq!(session.project.name, name); + let notices = session.notices_since(mark); + assert_eq!(notices.iter().filter(|n| !n.error).count(), 1); + assert!(notices.iter().any(|n| !n.error && n.text.starts_with("3 resource(s)"))); + assert!(notices.iter().all(|n| !n.text.contains("RGSYNTH_") && n.text != "Deleted 3 features")); + let diagnostic_losses = session.diagnostics.records().into_iter().filter(|r| r.message.starts_with("3 resource(s)")).count(); + assert_eq!(diagnostic_losses, 1); + let mark = session.notice_mark(); + session.guarded("Next command", false, |_| Ok(Value::Null)).unwrap(); + assert!(session.notices_since(mark).is_empty()); + } +} diff --git a/crates/engine/src/peer.rs b/crates/engine/src/peer.rs new file mode 100644 index 000000000..89bf1a6f5 --- /dev/null +++ b/crates/engine/src/peer.rs @@ -0,0 +1,300 @@ +//! Project exchange with a peer is an import, never an explicit saved-file reopen. A framing +//! page receives no credentials and cannot borrow the process's identical-text restore memory. + +use crate::{ + Project, Session, + durable::{self, Report}, +}; +use durable::{At, Durable, FieldMut, Kept, Leaf, Owner}; +use std::collections::{HashMap, HashSet}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +enum SlotKey { + Map { index: usize, name: String, at: rustgis_map::SlotAt }, + Area { index: usize, at: rustgis_map::SlotAt }, + Table(u32), + Video { presentation: usize, page: usize }, +} + +struct Slot<'a> { + key: SlotKey, + at: At, + value: &'a str, +} +struct Entry { + key: SlotKey, + public: String, + full: String, + ambiguous: bool, +} +pub(crate) struct Exchange { + generation: u64, + entries: Vec, +} +const MAX_ENTRIES: usize = 4096; +const MAX_BYTES: usize = 1 << 20; + +/// Enumerates slot identities before any normalization. Duplicate identities are retained in the +/// census: they must never be made unique by dropping or renumbering a hostile duplicate. +fn slots(p: &Project) -> Vec> { + let mut out = Vec::new(); + for (i, m) in p.maps.iter().enumerate() { + for slot in m.slots().into_iter().filter(|s| s.address) { + if let Some(at) = m.slot_identity(&slot.at) { + out.push(Slot { + key: SlotKey::Map { index: i, name: m.name.clone(), at }, + at: At { owners: vec![Owner::Map(i)], leaf: Leaf::Map(slot.at) }, + value: slot.value, + }); + } + } + } + for (i, area) in p.offline_areas.iter().enumerate() { + for slot in area.data.slots().into_iter().filter(|s| s.address) { + if let Some(at) = area.data.slot_identity(&slot.at) { + out.push(Slot { + key: SlotKey::Area { index: i, at }, + at: At { owners: vec![Owner::Area(i)], leaf: Leaf::Map(slot.at) }, + value: slot.value, + }); + } + } + } + for (i, table) in p.tables.iter().enumerate() { + if let Some(value) = &table.source { + out.push(Slot { key: SlotKey::Table(table.id), at: At { owners: vec![Owner::Table(i)], leaf: Leaf::Table }, value }); + } + } + for (i, pres) in p.presentations.iter().enumerate() { + for (page, p) in pres.pages.iter().enumerate() { + if let rustgis_map::presentation::PageKind::Video { source, .. } = &p.kind { + out.push(Slot { + key: SlotKey::Video { presentation: i, page }, + at: At { owners: vec![Owner::Presentation(i)], leaf: Leaf::Video(page) }, + value: source, + }); + } + } + } + out +} + +/// A wrapper's binding belongs to S3 connection resolution, never exchange retention. +fn wrapped(value: &str) -> bool { + let value = crate::sources::template_archive(value).map_or(value, |(_, v)| v); + value.trim_start().get(..11).is_some_and(|v| v.eq_ignore_ascii_case("rustgis-s3:")) +} + +/// Charges both forms and the complete identity payload, bounded before copying either form. +fn key_bytes(key: &SlotKey) -> usize { + let extra = match key { + SlotKey::Map { name, at, .. } => name.len().saturating_add(match at { + rustgis_map::SlotAt::Tile { id, .. } => id.len(), + _ => 0, + }), + SlotKey::Area { at: rustgis_map::SlotAt::Tile { id, .. }, .. } => id.len(), + _ => 0, + }; + std::mem::size_of::().saturating_add(extra) +} + +impl Exchange { + /// Recorded only after output rollback. Every entry is session-local and replaced per push. + fn record(p: &Project, generation: u64) -> (Self, u32) { + let mut entries: Vec = Vec::new(); + let mut recording = true; + let mut held = 0usize; + let mut omitted = 0u32; + for slot in slots(p) { + if wrapped(slot.value) { + continue; + } + let public = match durable::public_address(slot.value) { + Kept::Unchanged => continue, + Kept::Changed(s) => s, + Kept::Withheld(prefix) => prefix.into(), + }; + // Even an account omitted by the cap cannot make an earlier recorded public form + // unambiguous. After the first overflow, no later slot is recorded. + for e in entries.iter_mut().filter(|e| e.public == public) { + e.ambiguous |= e.full != slot.value; + } + let ambiguous = entries.iter().any(|e| e.public == public && e.ambiguous); + let bytes = slot.value.len().saturating_add(public.len()).saturating_add(key_bytes(&slot.key)); + if !recording || entries.len() >= MAX_ENTRIES || held.saturating_add(bytes) > MAX_BYTES { + recording = false; + omitted = omitted.saturating_add(1); + continue; + } + held += bytes; + entries.push(Entry { key: slot.key, public, full: slot.value.into(), ambiguous }); + } + (Self { generation, entries }, omitted) + } + + /// The pre-normalization census must contain exactly one byte-equal public slot. + fn incoming(&self, p: &Project) -> HashSet { + let census = census(p); + self.entries + .iter() + .filter(|e| census.get(&e.key).is_some_and(|(n, value)| *n == 1 && *value == e.public)) + .map(|e| e.key.clone()) + .collect() + } + + /// Checks last exchange, both incoming censuses, unique account and the exact current live + /// value in one synchronous replacement. Public URL or incoming id alone grants nothing. + fn retain(&self, live: &Project, incoming: &mut Project, raw: &HashSet) { + let current = census(live); + let normalized = census(incoming); + let mut records: HashMap<&SlotKey, usize> = HashMap::new(); + let mut accounts: HashMap<&str, Option<&str>> = HashMap::new(); + for e in &self.entries { + *records.entry(&e.key).or_default() += 1; + let account = accounts.entry(&e.public).or_insert(Some(&e.full)); + if *account != Some(e.full.as_str()) { + *account = None; + } + } + let edits: Vec<_> = slots(incoming) + .into_iter() + .filter_map(|slot| { + let e = self.entries.iter().find(|e| e.key == slot.key)?; + let unique = |census: &HashMap, expected: &str| { + census.get(&slot.key).is_some_and(|(n, value)| *n == 1 && *value == expected) + }; + (!e.ambiguous + && raw.contains(&slot.key) + && records.get(&slot.key) == Some(&1) + && accounts.get(e.public.as_str()) == Some(&Some(e.full.as_str())) + && unique(¤t, &e.full) + && unique(&normalized, &e.public)) + .then(|| (slot.at, e.full.clone())) + }) + .collect(); + drop(normalized); + for (at, full) in edits { + if let Some(FieldMut::Text(slot)) = incoming.field_mut(&at.owners, &at.leaf) { + *slot = full; + } + reopen(incoming, &at); + } + } +} + +/// A retained address must reopen its own reader, which normalization previously opened with +/// the public form. Only the exact retained owner/role is touched; S3 wrappers are excluded. +fn reopen(p: &mut Project, at: &At) { + let Some(owner) = at.owners.first() else { return }; + let map = match owner { + Owner::Map(i) => p.maps.get_mut(*i), + Owner::Area(i) => p.offline_areas.get_mut(*i).map(|a| &mut a.data), + _ => None, + }; + let Some(map) = map else { return }; + let Leaf::Map(rustgis_map::SlotPos::Layer { layer, role }) = &at.leaf else { return }; + let Some(layer) = map.layers.get_mut(*layer) else { return }; + match role { + rustgis_map::LayerRole::Streamed => { + if let Some(source) = &mut layer.source + && let Err(e) = source.reopen() + { + source.error = Some(rustgis_data::diagnostics::Redacted::new(&e).into_string()); + } + } + rustgis_map::LayerRole::Raster => { + if let Some(raster) = &mut layer.raster + && let Err(e) = raster.reload() + { + raster.error = Some(rustgis_data::diagnostics::Redacted::new(&e).into_string()); + } + } + _ => {} + } +} + +/// Values borrow the project, but identities include all duplicates; no normalization runs here. +fn census(p: &Project) -> HashMap { + let mut out = HashMap::new(); + for slot in slots(p) { + let entry = out.entry(slot.key).or_insert((0usize, slot.value)); + entry.0 = entry.0.saturating_add(1); + } + out +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Peer { + Bridge, + Embedder, +} + +impl Session { + /// Writes a credential-free project for a peer, without evicting explicit-save restoration. + /// Pending downloads require a structural draft; settled projects use the live journal. + pub fn peer_text(&mut self, peer: Peer) -> Result<(String, Report), String> { + let (text, mut report) = if self.project.ensure_imports_complete().is_err() { + let mut draft = self.project.clone(); + draft.omit_pending_imports(); + durable::output_owned(draft, text) + } else { + durable::output(&mut self.project, None, text) + }?; + self.exchange = if peer == Peer::Bridge { + let (record, omitted) = Exchange::record(&self.project, self.opened); + report.not_retained = omitted; + Some(record) + } else { + None + }; + Ok((text, report)) + } + + /// Replaces a project synchronously with peer/import semantics. No command runs with User + /// authority. Reset advances generation, clears item trust, history, pending work and active + /// connections; old approvals and continuations must bind to that generation. + pub fn open_peer_text(&mut self, peer: Peer, text: &str) -> Result { + let mut p = Project::from_peer_raw(text)?; + let exchange = self.exchange.as_ref().filter(|e| peer == Peer::Bridge && e.generation == self.opened); + let raw = exchange.map(|e| e.incoming(&p)); + crate::cmd::connections::clear_active(); + p.normalize(); + let mut report = Report::default(); + + if let (Some(exchange), Some(raw)) = (exchange, raw) { + exchange.retain(&self.project, &mut p, &raw); + } + let incoming = census(&p); + report.lost = slots(&self.project) + .into_iter() + .filter(|slot| { + !matches!(slot.key, SlotKey::Video { .. }) + && !matches!(durable::public_address(slot.value), Kept::Unchanged) + && !incoming.get(&slot.key).is_some_and(|(count, value)| *count == 1 && *value == slot.value) + }) + .count() + .try_into() + .unwrap_or(u32::MAX); + drop(incoming); + crate::cmd::project::reset_to(self, p); + self.path = None; + crate::cmd::connections::resolve_project(self); + crate::cmd::connections::note_waiting(self); + self.revision = self.revision.wrapping_add(1); + if peer == Peer::Bridge { + let (exchange, omitted) = Exchange::record(&self.project, self.opened); + self.exchange = Some(exchange); + report.not_retained = omitted; + } + Ok(report) + } +} + +/// The same opening bound applies to peers; no destination is written on a formatting failure. +fn text(p: &Project) -> Result { + let text = serde_json::to_string(p).map_err(|e| e.to_string())?; + if text.len() as u64 > crate::project::MAX_BYTES { + return Err("the project is too large to exchange".into()); + } + Ok(text) +} diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 80aa45d32..217965ef2 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -1,9 +1,6 @@ //! The project: maps, standalone tables and geoprocessing history, saved as JSON (`.rgis`). -use std::collections::hash_map::RandomState; -use std::hash::BuildHasher; use std::path::Path; -use std::sync::{Mutex, MutexGuard, OnceLock}; use rustgis_data::{Dataset, Feature, Field, GeometryType}; use rustgis_map::Map; @@ -58,69 +55,6 @@ pub const FORMAT: u32 = 1; /// Largest project file read or saved (512 MiB). pub const MAX_BYTES: u64 = 512 << 20; -/// The address a saved file keeps. Only web addresses can contain layer credentials. A tile -/// template (`pmtiles:…@archive`, `cog:…@archive`) is judged by its archive, and an address read -/// through a saved S3 connection by the address it wraps, the wrapper rebuilt around it (its -/// binding and endpoint are provenance, not secrets); a COG's look after either is kept. -fn saved_address(address: &str) -> Option { - use rustgis_data::s3_address; - if let Some((head, archive)) = crate::sources::template_archive(address) { - let (file, look) = split_look(archive); - return saved_address(file).map(|saved| format!("{head}{saved}{look}")); - } - if s3_address::is_wrapper(address) { - let (file, look) = split_look(address); - // One that doesn't parse stays as it is (and blocked): it names no connection. - let b = s3_address::parse(file).ok().flatten()?; - let inner = saved_address(b.url)?; - return s3_address::bind(b.binding, b.endpoint, &inner).ok().map(|saved| format!("{saved}{look}")); - } - let inner = address.strip_prefix(crate::render::rustgis_tiles::s3::ANONYMOUS).unwrap_or(address); - let (protocol, _) = inner.split_once("://")?; - if !["http", "https", "s3"].iter().any(|s| protocol.eq_ignore_ascii_case(s)) { - return None; - } - let (scheme, mut rest) = address.split_once("://")?; - let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); - if let Some(at) = rest.get(..end).and_then(|authority| authority.rfind('@')) { - rest = rest.get(at + 1..)?; - } - let rest = rest.split('#').next().unwrap_or(rest); - let mut out = format!("{scheme}://{rest}"); - rustgis_data::source::credentials::redact_query(&mut out); - (out != address).then_some(out) -} - -/// A COG address without its look (`…#bands=…`), and the look as written. -fn split_look(address: &str) -> (&str, &str) { - let file = crate::render::rustgis_tiles::cog::split(address).0; - (file, address.get(file.len()..).unwrap_or("")) -} - -/// Strips layer credentials, keeping originals in the visitor's order for the caller to restore. -/// Packaging calls this without registering the resulting text as a file this process wrote. -pub(crate) fn strip_addresses(addresses: Vec<&mut String>) -> Vec> { - addresses.into_iter().map(|a| saved_address(a).map(|redacted| std::mem::replace(a, redacted))).collect() -} - -struct Written { - len: usize, - hash: u64, - was: Vec>, -} - -const MAX_WRITTEN: usize = 32; - -fn kept() -> MutexGuard<'static, Vec> { - static KEPT: OnceLock>> = OnceLock::new(); - KEPT.get_or_init(Default::default).lock().unwrap_or_else(|e| e.into_inner()) -} - -fn identity(text: &str) -> (usize, u64) { - static KEY: OnceLock = OnceLock::new(); - (text.len(), KEY.get_or_init(RandomState::new).hash_one(text.as_bytes())) -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Project { #[serde(default = "format")] @@ -762,70 +696,8 @@ impl Project { Ok(()) } - /// The file keeps no passwords, keys, tokens or signatures in layer addresses. The open project - /// keeps them. This process remembers the 32 most recent distinct saved snapshots with stripped - /// addresses: opening identical plain project text restores them until that snapshot is evicted - /// or RustGIS closes. Packages and imports never restore them. - /// Refuses text larger than the opening limit, restoring live paths and addresses on failure. - pub fn saved_text(&mut self, dir: Option<&Path>) -> Result<(String, usize), String> { - self.ensure_imports_complete()?; - let was = strip_addresses(self.addresses_mut()); - let n = was.iter().flatten().count(); - let originals: Vec> = if let Some(d) = dir.and_then(Path::to_str) { - self.file_paths_mut() - .into_iter() - .map(|(p, _)| { - let relative = rustgis_map::paths::is_absolute_local(p) - .then(|| rustgis_map::paths::relative_to(d, p)) - .flatten() - // Keep the traversal stable during restoration, even for an unusually deep - // destination whose relative spelling exceeds the local-path size bound. - .filter(|r| rustgis_map::paths::is_local(r)); - relative.map(|r| std::mem::replace(p, r)) - }) - .collect() - } else { - Vec::new() - }; - let text = serde_json::to_string(self).map_err(|e| e.to_string()); - if !originals.is_empty() { - for ((p, _), original) in self.file_paths_mut().into_iter().zip(originals) { - if let Some(original) = original { - *p = original; - } - } - } - // Serialization can fail: put the live addresses back before handling its result. - for (address, original) in self.addresses_mut().into_iter().zip(&was) { - if let Some(original) = original { - *address = original.clone(); - } - } - let text = text?; - if text.len() as u64 > MAX_BYTES { - return Err(format!("the project is too large to save (maximum {} MiB)", MAX_BYTES >> 20)); - } - if n > 0 { - let (len, hash) = identity(&text); - let mut kept = kept(); - kept.retain(|entry| (entry.len, entry.hash) != (len, hash)); - if kept.len() >= MAX_WRITTEN { - kept.remove(0); - } - kept.push(Written { len, hash, was }); - } - Ok((text, n)) - } - - /// The project as an autosaved recovery copy keeps it: its text without passwords, keys, tokens - /// or signatures in layer addresses, from a copy of the project (autosave writes it off the - /// interface thread). Unlike [`Self::saved_text`], local file paths stay as the open project - /// holds them, absolute: the copy lives in the app's recovery folder and is opened from there, - /// not from the project's folder. Nor is it remembered as a file this process wrote, so - /// frequent copies can't push the user's own saves out of the snapshots that restore secrets. - /// Runtime import reservations are omitted while completed work remains recoverable. - /// Text too large to open again is refused. - pub fn recovery_text(mut self) -> Result { + /// Omits runtime-only reservations on an owned recovery or bridge draft. + pub(crate) fn omit_pending_imports(&mut self) { // A recovery copy preserves completed work without persisting runtime-only reservations. for map in self.maps.iter_mut().chain(self.offline_areas.iter_mut().map(|area| &mut area.data)) { let pending: std::collections::HashSet<_> = map @@ -840,28 +712,34 @@ impl Project { } map.tiles.retain(|tile| !tile.id.starts_with("geolibre-pending:")); } - strip_addresses(self.addresses_mut()); - let text = serde_json::to_string(&self).map_err(|e| e.to_string())?; + } + + /// Writes controlled durable fields and relative paths in one guarded transaction. Live + /// fields are restored on every return and native unwind; only explicit saves are remembered. + pub fn saved_text(&mut self, dir: Option<&Path>) -> Result<(String, crate::durable::Report), String> { + self.ensure_imports_complete()?; + crate::durable::output_remembered(self, dir.and_then(Path::to_str), project_text) + } + + /// The project as an autosaved recovery copy keeps it: its text without passwords, keys, tokens + /// or signatures in layer addresses, from a copy of the project (autosave writes it off the + /// interface thread). Unlike [`Self::saved_text`], local file paths stay as the open project + /// holds them, absolute: the copy lives in the app's recovery folder and is opened from there, + /// not from the project's folder. Nor is it remembered as a file this process wrote, so + /// frequent copies can't push the user's own saves out of the snapshots that restore secrets. + /// Runtime import reservations are omitted while completed work remains recoverable. + /// Text too large to open again is refused. + pub fn recovery_text(mut self) -> Result { + self.omit_pending_imports(); + let (text, _) = crate::durable::output_owned(self, project_text)?; if text.len() as u64 > MAX_BYTES { return Err("the project is too large to keep a recovery copy".into()); } Ok(text) } - fn restore_addresses(&mut self, text: &str) { - let id = identity(text); - let kept = kept(); - let Some(entry) = kept.iter().find(|entry| (entry.len, entry.hash) == id) else { return }; - for (address, original) in self.addresses_mut().into_iter().zip(&entry.was) { - if let Some(full) = original - && saved_address(full).as_deref() == Some(address.as_str()) - { - *address = full.clone(); - } - } - } - - pub fn save(&mut self, path: &Path) -> Result { + /// Writes checked durable text atomically and returns counted losses from the guarded output. + pub fn save(&mut self, path: &Path) -> Result { let dir = std::path::absolute(path).ok().and_then(|p| p.parent().map(Path::to_path_buf)); let (text, stripped) = self.saved_text(dir.as_deref())?; rustgis_data::io::write_atomic(path, text.as_bytes()).map_err(|e| format!("could not save {}: {e}", path.display()))?; @@ -870,15 +748,22 @@ impl Project { /// Reads a project from its JSON text (a browser upload). pub fn from_json(text: &str) -> Result { - Self::read_json(text, true, None) + Self::read_json(text, true, None, true) } /// Reads imported JSON (including package contents) without restoring a saved file's secrets. pub(crate) fn from_import_json(text: &str, dir: Option<&Path>) -> Result { - Self::read_json(text, false, dir) + Self::read_json(text, false, dir, true) } - fn read_json(text: &str, restore: bool, dir: Option<&Path>) -> Result { + /// Peer census must precede normalization: duplicate ids cannot become proof of a unique + /// matching slot after the normalizer renumbers or drops them. Saved-text restore is absent. + pub(crate) fn from_peer_raw(text: &str) -> Result { + Self::read_json(text, false, None, false) + } + + /// Checks project size and version; explicit saves may restore exact fields before normalization. + fn read_json(text: &str, restore: bool, dir: Option<&Path>, normalize: bool) -> Result { if text.len() as u64 > MAX_BYTES { return Err("the project file is too large".into()); } @@ -901,7 +786,7 @@ impl Project { return Err(format!("this project was saved by a newer RustGIS (format {}). Update RustGIS to open it.", p.format)); } if restore { - p.restore_addresses(text); + crate::durable::restore(&mut p, text); } // A binding is this computer's runtime token for a login: one in a file is no authority // to use any login here. @@ -909,7 +794,9 @@ impl Project { if let Some(d) = dir.and_then(Path::to_str) { p.resolve_paths(d); } - p.normalize(); + if normalize { + p.normalize(); + } Ok(p) } @@ -919,7 +806,7 @@ impl Project { let bytes = read_file(path, MAX_BYTES)?; let text = String::from_utf8(bytes).map_err(|_| format!("{} isn't a RustGIS project", path.display()))?; let dir = std::path::absolute(path).ok().and_then(|p| p.parent().map(Path::to_path_buf)); - Self::read_json(&text, true, dir.as_deref()) + Self::read_json(&text, true, dir.as_deref(), true) } } @@ -974,3 +861,178 @@ fn unique_names<'a>(names: impl Iterator) { } } } + +// Durable ownership is classified here so private fields cannot disappear from the inventory. +use crate::durable::{Durable, FieldMut, Leaf, Owner, Prepare}; + +impl Durable for Project { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + let Self { + format: _, + name: _, + maps, + active_map: _, + tables, + next_table_id: _, + gp_history, + layouts: _, + folders: _, + styles: _, + ramps: _, + tasks, + gp_env: _, + models, + workspaces: _, + link_views: _, + thumbnail: _, + jobs: _, + offline_areas, + reports: _, + presentations, + notebooks, + dl_models: _, + shared: _, + data_stores: _, + } = self; + for (i, v) in maps.iter().enumerate() { + p.child(Owner::Map(i), v); + } + for (i, v) in offline_areas.iter().enumerate() { + let OfflineArea { name: _, map: _, extent: _, created: _, synced: _, data } = v; + p.child(Owner::Area(i), data); + } + for (i, v) in tables.iter().enumerate() { + p.child(Owner::Table(i), v); + } + for (i, v) in tasks.iter().enumerate() { + p.child(Owner::Task(i), v); + } + for (i, v) in models.iter().enumerate() { + p.child(Owner::Model(i), v); + } + for (i, v) in gp_history.iter().enumerate() { + p.child(Owner::History(i), v); + } + for (i, v) in notebooks.iter().enumerate() { + p.child(Owner::Notebook(i), v); + } + for (i, v) in presentations.iter().enumerate() { + p.child(Owner::Presentation(i), v); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + let (owner, rest) = owners.split_first()?; + match owner { + Owner::Map(i) => self.maps.get_mut(*i)?.field_mut(rest, leaf), + Owner::Area(i) => self.offline_areas.get_mut(*i)?.data.field_mut(rest, leaf), + Owner::Table(i) => self.tables.get_mut(*i)?.field_mut(rest, leaf), + Owner::Task(i) => self.tasks.get_mut(*i)?.field_mut(rest, leaf), + Owner::Model(i) => self.models.get_mut(*i)?.field_mut(rest, leaf), + Owner::History(i) => self.gp_history.get_mut(*i)?.field_mut(rest, leaf), + Owner::Notebook(i) => self.notebooks.get_mut(*i)?.field_mut(rest, leaf), + Owner::Presentation(i) => self.presentations.get_mut(*i)?.field_mut(rest, leaf), + Owner::Graph => None, + } + } +} + +impl Durable for Table { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + let Self { id: _, dataset } = self; + let Dataset { + name: _, + geometry_type: _, + fields: _, + features: _, + source: provenance, + crs: _, + crs_incomplete: _, + zm_dropped: _, + windowed: _, + } = dataset; + if let Some(v) = provenance { + p.string(Leaf::Table, v, true, rustgis_map::paths::is_local(v)); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + if owners.is_empty() && *leaf == Leaf::Table { self.dataset.source.as_mut().map(FieldMut::Text) } else { None } + } +} + +impl Durable for Task { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + let Self { name: _, description: _, steps } = self; + for (i, TaskStep { title: _, instructions: _, command, params }) in steps.iter().enumerate() { + p.command(Leaf::Task(i), command, params); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + if !owners.is_empty() { + return None; + } + let Leaf::Task(i) = leaf else { return None }; + Some(FieldMut::Json(&mut self.steps.get_mut(*i)?.params)) + } +} + +impl Durable for Model { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + let Self { name: _, steps, graph } = self; + for (i, ModelStep { tool, params }) in steps.iter().enumerate() { + p.tool(Leaf::ModelStep(i), tool, params); + } + if let Some(g) = graph { + p.child(Owner::Graph, g); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + if let Some((Owner::Graph, rest)) = owners.split_first() { + return self.graph.as_mut()?.field_mut(rest, leaf); + } + if !owners.is_empty() { + return None; + } + let Leaf::ModelStep(i) = leaf else { return None }; + Some(FieldMut::Json(&mut self.steps.get_mut(*i)?.params)) + } +} + +impl Durable for ModelGraph { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + let Self { nodes, edges: _ } = self; + for (i, ModelNode { id: _, kind: _, x: _, y: _, layer: _, tool, params, name: _ }) in nodes.iter().enumerate() { + p.tool(Leaf::ModelNode(i), tool.as_deref().unwrap_or(""), params); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + if !owners.is_empty() { + return None; + } + let Leaf::ModelNode(i) = leaf else { return None }; + Some(FieldMut::Json(&mut self.nodes.get_mut(*i)?.params)) + } +} + +/// Enforces the opening bound inside the formatter, before any destination is replaced. +fn project_text(p: &Project) -> Result { + let text = serde_json::to_string(p).map_err(|e| e.to_string())?; + if text.len() as u64 > MAX_BYTES { + return Err(format!("the project is too large to save (maximum {} MiB)", MAX_BYTES >> 20)); + } + Ok(text) +} diff --git a/crates/engine/src/project_log.rs b/crates/engine/src/project_log.rs index 12ac74c07..cec01e8d0 100644 --- a/crates/engine/src/project_log.rs +++ b/crates/engine/src/project_log.rs @@ -6,6 +6,22 @@ use serde_json::Value; use crate::{CmdResult, Session}; +/// A redacted output projection. Never use it as executable parameters or approval identity. +pub struct LogCopy(Value); + +impl LogCopy { + /// Consumes the output-only wrapper for serialization. + pub fn into_log_value(self) -> Value { + self.0 + } +} + +/// The session log's bounded, document-aware parameter policy, shared with the agent audit. +/// The source value is never mutated and remains the only input to execution and planning. +pub fn log_params(params: &Value) -> LogCopy { + LogCopy(kept_params(params)) +} + /// Now, in milliseconds since 1970 (UTC). pub(crate) fn now_ms() -> u64 { web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0) @@ -126,7 +142,17 @@ fn embedded_redacted(v: &Value, depth: usize) -> Value { Value::String(sql_secrets_removed(&t)) } Value::Array(a) if depth < 8 => Value::Array(a.iter().map(|x| embedded_redacted(x, depth + 1)).collect()), - Value::Object(o) if depth < 8 => Value::Object(o.iter().map(|(k, x)| (k.clone(), embedded_redacted(x, depth + 1))).collect()), + Value::Object(o) if depth < 8 => Value::Object( + o.iter() + .map(|(k, x)| { + // Preserve History's plain-key refinement; the log-only fields share diagnostics' + // canonical list. Clipping already bounds recursion and text before this pass. + let sensitive = + rustgis_data::address::normalized_name(k).as_deref() != Some("key") && rustgis_data::address::log_sensitive(k); + (k.clone(), if sensitive { Value::String(String::new()) } else { embedded_redacted(x, depth + 1) }) + }) + .collect(), + ), other => other.clone(), } } diff --git a/crates/engine/src/roles.rs b/crates/engine/src/roles.rs new file mode 100644 index 000000000..3a1df9f9f --- /dev/null +++ b/crates/engine/src/roles.rs @@ -0,0 +1,374 @@ +//! Replayable parameters are cleaned by role, never by recursively deleting secret-named data +//! properties. Unknown nested content and foreign notebook cells remain opaque, with honest +//! notices. Logs deliberately have a separate, stricter policy. + +use std::collections::HashSet; +use std::sync::OnceLock; + +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; + +use crate::durable::{self, Durable, Kept, Report}; + +const WITHHELD: &str = "[parameters withheld]"; +const NESTED: &str = "[withheld: nested too deep]"; +const DATA: &[&str] = &[ + "text", + "data", + "values", + "value", + "attributes", + "properties", + "features", + "feature", + "geometry", + "rows", + "expression", + "where", + "html", + "markdown", + "caption", + "title", + "notes", + "description", + "label", + "name", + "field", + "fields", +]; +const ADDRESS: &[&str] = + &["url", "urls", "href", "source", "template", "tiles", "store", "endpoint", "address", "uri", "connection", "database"]; +const CONFIG: &[&str] = &["headers", "options", "auth", "connection", "source", "sources"]; + +/// Registered commands are classified once; plugin commands keep their nested content opaque. +fn known(id: &str) -> bool { + static IDS: OnceLock> = OnceLock::new(); + IDS.get_or_init(|| crate::cmd::all().into_iter().map(|c| c.id).collect()).contains(id) +} + +/// A conservative bounded read-only check, used solely to announce unchecked/withheld content. +fn suspicious(v: &Value) -> bool { + crate::sources::holds_secret(v) + || v.as_str().is_some_and(|s| { + s.split(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '%'))) + .take(65_536) + .any(|word| !word.is_empty() && rustgis_data::address::secret_key(word)) + }) +} + +/// Keeps invalid executable content invalid while withholding credential-bearing text. +fn invalid(value: &Value, report: &mut Report) -> Option { + if suspicious(value) { + report.withheld = report.withheld.saturating_add(1); + Some(Value::String(WITHHELD.into())) + } else { + None + } +} + +/// Cleans one command without treating parameter JSON strings as executor envelopes. +pub(crate) fn command(id: &str, params: &Value, depth: usize) -> (Option, Report) { + let mut report = Report::default(); + let changed = object(id, params, depth, 0, known(id), &mut report); + (changed, report) +} + +/// Tools use command roles too, including the data-valued `input`/`values` parameters. +pub(crate) fn tool(id: &str, params: &Value, depth: usize) -> (Option, Report) { + let mut report = Report::default(); + let builtin = rustgis_analysis::TOOLS.iter().any(|t| t.id == id); + let changed = object("", params, depth, 0, builtin, &mut report); + (changed, report) +} + +/// Lazily copies only changed owning parameter objects. Configuration recursion is bounded. +fn object(id: &str, value: &Value, depth: usize, config_depth: usize, registered: bool, report: &mut Report) -> Option { + if value.is_null() { + return None; + } + let Some(obj) = value.as_object() else { return invalid(value, report) }; + if config_depth >= 8 { + return invalid(value, report); + } + let mut replacement = None; + let mut unchecked = false; + for (key, v) in obj { + unchecked |= !matches!(durable::public_address(key), Kept::Unchanged); + let k = rustgis_data::address::normalized_name(key).unwrap_or_else(|| key.to_ascii_lowercase()); + let executable = registered + && matches!( + (id, k.as_str()), + ("gp.run", "params") + | ("script.run", "script") + | ("task.new", "steps") + | ("model.new" | "model.save" | "model.run" | "model.validate" | "model.export", "graph") + | ("model.new", "steps") + ); + let document = registered + && k == "text" + && matches!(id, "project.openText" | "map.import" | "task.import" | "notebook.open" | "presentation.import" | "model.import"); + let clean = if executable || document { + if depth >= 4 { + report.withheld = report.withheld.saturating_add(1); + Some(Value::String(NESTED.into())) + } else { + envelope(id, &k, v, obj, depth + 1, report) + } + } else if rustgis_data::address::secret_key(key) + && (k != "key" + || crate::sources::looks_like_a_key(v.as_str().unwrap_or("")) + || obj.iter().any(|(other, value)| other != key && !DATA.contains(&other.as_str()) && address_like(value))) + { + match v { + Value::String(s) if !s.is_empty() => Some(Value::String(String::new())), + Value::Number(_) => Some(Value::String(String::new())), + _ => None, + } + } else if DATA.contains(&k.as_str()) { + None + } else if registered && id.starts_with("sql.") && matches!(k.as_str(), "sql" | "query") { + v.as_str().and_then(|s| { + let clean = crate::project_log::sql_secrets_removed(s); + (clean != s).then_some(Value::String(clean)) + }) + } else if ADDRESS.contains(&k.as_str()) || address_like(v) { + if v.is_object() && registered && CONFIG.contains(&k.as_str()) { + object("", v, depth, config_depth + 1, true, report) + } else { + address(v, report) + } + } else if registered && CONFIG.contains(&k.as_str()) && v.is_object() { + object("", v, depth, config_depth + 1, true, report) + } else { + unchecked |= suspicious(v); + None + }; + if !registered && !DATA.contains(&k.as_str()) && (v.is_object() || v.is_array()) { + unchecked |= suspicious(v); + } + if let Some(clean) = clean.filter(|clean| clean != v) { + replacement.get_or_insert_with(|| obj.clone()).insert(key.clone(), clean); + } + } + if unchecked { + report.unchecked = report.unchecked.saturating_add(1); + } + replacement.map(Value::Object) +} + +/// Recognition precedes parsing, including malformed wrappers and differently cased prefixes. +fn address_like(v: &Value) -> bool { + match v { + Value::String(s) => { + let s = s.trim_start(); + ["http:", "https:", "s3:", "rustgis-anonymous:", "rustgis-s3:", "pmtiles:", "cog:", "postgres:", "postgresql:", "pg:"] + .iter() + .any(|p| s.get(..p.len()).is_some_and(|s| s.eq_ignore_ascii_case(p))) + } + Value::Array(a) => a.iter().any(|v| v.is_string() && address_like(v)), + _ => false, + } +} + +/// Array addresses are string leaves only; data and nested opaque objects are never walked. +fn address(v: &Value, report: &mut Report) -> Option { + match v { + Value::String(s) => match durable::public_address(s) { + Kept::Unchanged => None, + Kept::Changed(s) => Some(Value::String(s)), + Kept::Withheld(prefix) if prefix != s => { + report.withheld = report.withheld.saturating_add(1); + Some(Value::String(prefix.into())) + } + Kept::Withheld(_) => None, + }, + Value::Array(a) => { + let mut changed = None; + for (i, v) in a.iter().enumerate() { + if v.is_string() + && let Some(clean) = address(v, report) + && let Some(slot) = changed.get_or_insert_with(|| a.clone()).get_mut(i) + { + *slot = clean; + } + } + changed.map(Value::Array) + } + _ => None, + } +} + +/// Explicit executable and document roles are the only route into nested executable content. +fn envelope(id: &str, key: &str, v: &Value, siblings: &serde_json::Map, depth: usize, report: &mut Report) -> Option { + match (id, key) { + ("gp.run", "params") => { + let (clean, r) = tool(siblings.get("tool").and_then(Value::as_str).unwrap_or(""), v, depth); + report.merge(&r); + clean + } + ("script.run", "script") => match v.as_str() { + Some(s) => { + let (clean, r) = script(s, depth); + report.merge(&r); + clean.map(Value::String) + } + None => invalid(v, report), + }, + ("task.new", "steps") => typed_value::(v, depth, report), + ("model.new", "steps") => typed_value::(v, depth, report), + (_, "graph") => typed_value::(v, depth, report), + (_, "text") => document(id, v, depth, report), + _ => None, + } +} + +/// A typed envelope is prepared on its own copy, preserving the live outer params as a whole. +fn typed_value(v: &Value, depth: usize, report: &mut Report) -> Option { + let Ok(parsed) = serde_json::from_value::(v.clone()) else { return invalid(v, report) }; + let Ok((clean, r)) = durable::output_owned_changed(parsed, depth, |v, changed| { + if changed { serde_json::to_value(v).map(Some).map_err(|e| e.to_string()) } else { Ok(None) } + }) else { + return invalid(v, report); + }; + report.merge(&r); + clean +} + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(transparent)] +struct VecTask(Vec); + +impl Durable for VecTask { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut durable::Prepare<'_>) { + for (i, st) in self.0.iter().enumerate() { + p.command(durable::Leaf::Task(i), &st.command, &st.params); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[durable::Owner], leaf: &durable::Leaf) -> Option> { + if !owners.is_empty() { + return None; + } + let durable::Leaf::Task(i) = leaf else { return None }; + Some(durable::FieldMut::Json(&mut self.0.get_mut(*i)?.params)) + } +} + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(transparent)] +struct VecSteps(Vec); + +impl Durable for VecSteps { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut durable::Prepare<'_>) { + for (i, st) in self.0.iter().enumerate() { + p.tool(durable::Leaf::ModelStep(i), &st.tool, &st.params); + } + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[durable::Owner], leaf: &durable::Leaf) -> Option> { + if !owners.is_empty() { + return None; + } + let durable::Leaf::ModelStep(i) = leaf else { return None }; + Some(durable::FieldMut::Json(&mut self.0.get_mut(*i)?.params)) + } +} + +/// Reads documents at the existing reader bound, without normalization, trust or restore-cache +/// access. Unchanged documents keep their exact text; only changed typed copies are serialized. +fn document(id: &str, v: &Value, depth: usize, report: &mut Report) -> Option { + let Some(text) = v.as_str().filter(|s| s.len() as u64 <= crate::project::MAX_BYTES) else { return invalid(v, report) }; + let result = match id { + "project.openText" => text_document::(text, depth), + "map.import" => { + if serde_json::from_str::(text).is_ok() { + text_document::(text, depth) + } else { + text_document::(text, depth) + } + } + "task.import" => text_document::(text, depth), + "presentation.import" => text_document::(text, depth), + "notebook.open" => crate::cmd::notebook::durable_document(text, depth), + "model.import" => model_document(text, depth), + _ => return None, + }; + match result { + Ok((clean, r)) => { + let changed = clean != text; + report.merge(&r); + changed.then_some(Value::String(clean)) + } + Err(_) => invalid(v, report), + } +} + +/// Uses the owning type's ordinary deserializer, without losing unchanged text representation. +fn text_document(text: &str, depth: usize) -> Result<(String, Report), String> { + let parsed = serde_json::from_str::(text).map_err(|e| e.to_string())?; + durable::output_owned_changed(parsed, depth, |value, changed| { + if changed { serde_json::to_string(value).map_err(|e| e.to_string()) } else { Ok(text.into()) } + }) +} + +/// Model import has two envelopes; keeps the envelope's other fields and cleans only its model. +fn model_document(text: &str, depth: usize) -> Result<(String, Report), String> { + let mut v: Value = serde_json::from_str(text).map_err(|e| e.to_string())?; + let mut report = Report::default(); + let Some(obj) = v.as_object_mut() else { return Err("not a model".into()) }; + let (key, graph) = if obj.contains_key("graph") { ("graph", true) } else { ("steps", false) }; + let Some(field) = obj.get(key) else { return Err("not a model".into()) }; + let clean = if graph { + typed_value::(field, depth, &mut report) + } else { + typed_value::(field, depth, &mut report) + }; + if let Some(clean) = clean { + obj.insert(key.into(), clean); + Ok((v.to_string(), report)) + } else { + Ok((text.into(), report)) + } +} + +/// Uses the executor's line parser, preserving comments, blank lines and unchanged formatting. +pub(crate) fn script(source: &str, depth: usize) -> (Option, Report) { + let mut report = Report::default(); + let mut out = None; + let mut offset = 0usize; + for line in source.split_inclusive('\n') { + let ending = if line.ends_with("\r\n") { + "\r\n" + } else if line.ends_with('\n') { + "\n" + } else { + "" + }; + let replacement = match crate::script_line(line) { + None => None, + Some(Ok((id, params))) => { + let (clean, r) = command(&id, ¶ms, depth); + report.merge(&r); + clean.map(|params| format!("{id} {params}{ending}")) + } + Some(Err(_)) if suspicious(&Value::String(line.into())) => { + report.withheld = report.withheld.saturating_add(1); + let id = line.trim_start().split(char::is_whitespace).next().unwrap_or(""); + Some(format!("{id} {WITHHELD}{ending}")) + } + Some(Err(_)) => None, + }; + if let Some(clean) = replacement { + report.lines = report.lines.saturating_add(1); + out.get_or_insert_with(|| source.get(..offset).unwrap_or("").to_string()).push_str(&clean); + } else if let Some(out) = &mut out { + out.push_str(line); + } + offset += line.len(); + } + (out, report) +} diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 998d21d5f..971109616 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -160,7 +160,13 @@ pub fn clean(mut s: Source) -> Result { if !KINDS.contains(&s.kind.as_str()) { return Err(format!("unknown kind '{}' ({})", s.kind, KINDS.join(", "))); } - s.address = redact(s.address.trim()); + let address = s.address.trim(); + // A display-only prefix is safe to show, but cannot be used as a saved source. + s.address = match crate::durable::public_address(address) { + crate::durable::Kept::Withheld(_) => return Err("a source needs an address (a path, web address or connection)".into()), + crate::durable::Kept::Changed(public) => public, + crate::durable::Kept::Unchanged => redact(address), + }; // Provenance has its own structural bound; it must not consume the destination's limit. let (head, archive) = template_archive(&s.address).unwrap_or(("", s.address.as_str())); let bounded = match rustgis_data::s3_address::parse(archive) { @@ -344,15 +350,20 @@ pub(crate) fn template_archive(address: &str) -> Option<(&str, &str)> { /// saved S3 connection is redacted inside (its inner address), and its length-prefixed form /// rebuilt; a tile template, in its archive. pub fn redact(address: &str) -> String { - if let Some((head, archive)) = template_archive(address) - && rustgis_data::s3_address::is_wrapper(archive) - { - return format!("{head}{}", redact(archive)); - } - if let Ok(Some(b)) = rustgis_data::s3_address::parse(address) - && let Ok(rebuilt) = rustgis_data::s3_address::bind(b.binding, b.endpoint, &redact(b.url)) + match crate::durable::public_address(address) { + crate::durable::Kept::Changed(s) => return s, + crate::durable::Kept::Withheld(prefix) => return prefix.into(), + crate::durable::Kept::Unchanged => {} + } + // A clean recognized wrapper or database label already passed the shared owner. Keep its + // layout, including a COG look; the generic scheme fallback is for older connection forms. + if template_archive(address).is_some() + || rustgis_data::s3_address::is_wrapper(address) + || ["rustgis-anonymous:", "postgres:", "postgresql:", "pg:"] + .iter() + .any(|p| address.trim_start().get(..p.len()).is_some_and(|s| s.eq_ignore_ascii_case(p))) { - return rebuilt; + return address.to_string(); } if let Some((public, _)) = rustgis_data::source::myconn::split_password(address) { return public; @@ -360,13 +371,6 @@ pub fn redact(address: &str) -> String { if let Some((public, _)) = rustgis_data::source::msconn::split_password(address) { return public; } - let lower = address.to_ascii_lowercase(); - // A PostgreSQL connection (`postgres:` or `postgresql:` keyword strings, GDAL's `PG:`, URIs): - // libpq's own rules (quoted values, escapes, `key = value`), without its secrets. - if ["postgres:", "postgresql:", "pg:"].iter().any(|p| lower.starts_with(p)) { - // One that can't be taken apart isn't kept (an empty address, refused by `clean`). - return rustgis_data::source::pgconn::public_label(address).unwrap_or_default(); - } let Some((scheme, rest)) = address.split_once("://") else { return address.to_string() }; // The user and password go first, before anything splits the address: a password may hold // an unencoded `/`, `?`, `#` or `@`. They end at the last `@` before the first `/`, or (a `/` @@ -746,9 +750,15 @@ MIIE for f in ["#Access_Token=t", "#sig=s", "#password=p", "#id_token=i", "#map=4/1/2"] { assert_eq!(redact(&format!("https://h.example/x.fgb{f}")), "https://h.example/x.fgb", "{f}"); } - // A connection with nothing to take apart isn't kept. - for bare in ["postgres:dbname", "PG:", "postgres:password='x"] { - assert!(clean(Source { kind: "database".into(), address: bare.into(), ..Source::default() }).is_err(), "{bare}"); + // Unreadable connections, web addresses and wrappers cannot become saved sources. + for (kind, bare) in [ + ("database", "postgres:dbname"), + ("database", "PG:"), + ("database", "postgres:password='x"), + ("url", "https:///path?token=RGSYNTH_unusable_source_20261104"), + ("tiles", "pmtiles:unreadable"), + ] { + assert!(clean(Source { kind: kind.into(), address: bare.into(), ..Source::default() }).is_err(), "{bare}"); } assert_eq!( redact("https://acct.blob.core.windows.net/c/a.tif?se=2026&sp=r&sv=1&sig=zzz"), diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 1d3f4e273..e897563eb 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5,6 +5,9 @@ use crate::{Project, Session, sample}; #[path = "tests/task_reentry.rs"] mod task_reentry; +#[path = "tests/credential_output.rs"] +mod credential_output; + #[cfg(not(target_arch = "wasm32"))] #[path = "tests/remembered_database.rs"] mod remembered_database; @@ -9145,7 +9148,7 @@ fn jupyter_python_notebooks_survive_open_and_save() { let nb = s.project.notebooks.last().unwrap(); let _ = (nb.language_label(), nb.cells.iter().map(crate::cmd::notebook::Cell::images).count()); let i = s.project.notebooks.len() - 1; - assert!(crate::cmd::notebook::notebook_file(&s, i, true).is_ok(), "{junk}"); + assert!(crate::cmd::notebook::notebook_file(&mut s, i, true).is_ok(), "{junk}"); s.run("notebook.remove", &json!({"notebook": i})).unwrap(); } } @@ -9156,7 +9159,7 @@ fn jupyter_python_notebooks_survive_open_and_save() { s.run("notebook.open", &json!({"text": cmds.to_string()})).unwrap(); s.run("notebook.run", &json!({"cell": 0})).unwrap(); let i = s.project.notebooks.len() - 1; - let saved: Value = serde_json::from_slice(&crate::cmd::notebook::notebook_file(&s, i, true).unwrap()).unwrap(); + let saved: Value = serde_json::from_slice(&crate::cmd::notebook::notebook_file(&mut s, i, true).unwrap()).unwrap(); let cell = &saved["cells"][0]; assert_eq!(cell["id"], "go"); assert_eq!(cell["execution_count"], Value::Null); @@ -11311,35 +11314,31 @@ fn adding_a_collection_of_3d_tiles_says_why_it_is_left_out() { fn an_arcgis_token_in_a_service_address_reaches_its_layers() { use crate::render::rustgis_tiles::arcgis_auth; let mut s = Session::new(Project::default()); - let base = "https://secure914.example.org/server/rest/services/Roads/MapServer"; + let base = "https://secure914.invalid/server/rest/services/Roads/MapServer"; let doc = r#"{"currentVersion": 11, "mapName": "Roads", "capabilities": "Map,Query,Data", "singleFusedMapCache": true, "tileInfo": {"rows": 256, "spatialReference": {"wkid": 102100}, "lods": [{"level": 0, "resolution": 156543.033928041}]}, "layers": [{"id": 0, "name": "Highways"}]}"#; - let v = s.run("service.layers", &json!({"url": format!("{base}?f=json&token=secret.Token-914"), "text": doc})).unwrap(); - assert!(!v.to_string().contains("secret.Token-914"), "{v}"); + let v = s.run("service.layers", &json!({"url": format!("{base}?f=json&%2574oken=RGSYNTH_3"), "text": doc})).unwrap(); + assert!(!v.to_string().contains("RGSYNTH_3"), "{v}"); let tiles = format!("{base}/tile/0/0/0"); let query = format!("{base}/0/query?where=1%3D1&f=geojson"); - assert_eq!(arcgis_auth::authorization(&tiles).as_deref(), Some("Bearer secret.Token-914")); - assert_eq!(arcgis_auth::authorization(&query).as_deref(), Some("Bearer secret.Token-914")); + assert_eq!(arcgis_auth::authorization(&tiles).as_deref(), Some("Bearer RGSYNTH_3")); + assert_eq!(arcgis_auth::authorization(&query).as_deref(), Some("Bearer RGSYNTH_3")); // Another server never gets it. - assert_eq!(arcgis_auth::authorization("https://other914.example.org/server/rest/services/Roads/MapServer/tile/0/0/0"), None); + assert_eq!(arcgis_auth::authorization("https://other914.invalid/server/rest/services/Roads/MapServer/tile/0/0/0"), None); // Added, the tile layer keeps the address without the token. let layer = v["layers"].as_array().unwrap().iter().find(|l| l["offer"]["type"] == "tiles").unwrap()["name"].as_str().unwrap().to_string(); - s.run("service.add", &json!({"url": format!("{base}?token=secret.Token-914"), "layer": layer, "text": doc})).unwrap(); + s.run("service.add", &json!({"url": format!("{base}?token=RGSYNTH_3"), "layer": layer, "text": doc})).unwrap(); let project = format!("{:?}", s.project); - assert!(project.contains("secure914.example.org") && !project.contains("secret.Token-914")); + assert!(project.contains("secure914.invalid") && !project.contains("RGSYNTH_3")); // Listed and forgotten by arcgis.status and arcgis.signOut. let status = s.run("arcgis.status", &json!({})).unwrap(); assert!( - status["sessions"] - .as_array() - .unwrap() - .iter() - .any(|x| x["portal"] == "https://secure914.example.org/server" && x["kind"] == "token") + status["sessions"].as_array().unwrap().iter().any(|x| x["portal"] == "https://secure914.invalid/server" && x["kind"] == "token") ); - assert!(!status.to_string().contains("secret.Token-914")); - s.run("arcgis.signOut", &json!({"portal": "https://secure914.example.org/server/"})).unwrap(); + assert!(!status.to_string().contains("RGSYNTH_3")); + s.run("arcgis.signOut", &json!({"portal": "https://secure914.invalid/server/"})).unwrap(); assert_eq!(arcgis_auth::authorization(&tiles), None); // Junk is refused rather than taken for "every session". for junk in [json!({"portal": 7}), json!({"portal": [true]}), json!({"portal": {"a": 1}})] { @@ -15896,14 +15895,20 @@ fn layer_address_redaction_preserves_templates_and_local_paths() { ("upload:map?token=local", "upload:map?token=local"), ("postgres:host=localhost dbname=gis", "postgres:host=localhost dbname=gis"), ]; - let mut addresses: Vec = cases.iter().map(|(a, _)| a.to_string()).collect(); - crate::project::strip_addresses(addresses.iter_mut().collect()); - for (address, (_, expected)) in addresses.iter().zip(cases) { + for (original, expected) in cases { + let address = match crate::durable::public_address(original) { + crate::durable::Kept::Unchanged => original.to_owned(), + crate::durable::Kept::Changed(address) => address, + crate::durable::Kept::Withheld(_) => panic!("valid address was withheld: {original}"), + }; assert_eq!(address, expected); } let prefix = crate::render::rustgis_tiles::s3::ANONYMOUS; - let mut anonymous = format!("{prefix}https://bucket.example/a?token=secret"); - crate::project::strip_addresses(vec![&mut anonymous]); + let crate::durable::Kept::Changed(anonymous) = + crate::durable::public_address(&format!("{prefix}https://bucket.example/a?token=secret")) + else { + panic!("anonymous address was not redacted") + }; assert_eq!(anonymous, format!("{prefix}https://bucket.example/a")); } @@ -16079,40 +16084,74 @@ fn recovery_copies_keep_no_layer_tokens_and_restore_none() { #[test] fn saved_projects_keep_no_layer_tokens() { let mut s = world(); - let url = "https://gis.example/arcgis/rest/services/a/MapServer/tile/{z}/{y}/{x}?token=s3cret"; + let url = "https://gis.example/arcgis/rest/services/a/MapServer/tile/{z}/{y}/{x}?token=s3cret-tile"; // Its legend, built from the service's address, with the key in it too. - let legend = "https://gis.example/wms?key=l3g&SERVICE=WMS&REQUEST=GetLegendGraphic&LAYER=a"; + let legend = "https://gis.example/wms?key=l3g-secret&SERVICE=WMS&REQUEST=GetLegendGraphic&LAYER=a"; s.run("map.addTiles", &json!({"url":url,"name":"Secured","legend":legend})).unwrap(); let layer = s.map().layers[0].id; - s.map_mut().layers[0].dataset.source = Some("https://u:pw@gis.example/f?f=json&access_token=tok2".into()); + s.map_mut().layers[0].dataset.source = Some("https://u:pw-secret@gis.example/f?f=json&access_token=tok2-secret".into()); s.map_mut().service_groups.push(rustgis_map::ServiceGroup { id: 123, name: "Protected service".into(), - url: "https://gis.example/wfs?key=k3y&map=m".into(), + url: "https://gis.example/wfs?key=k3y-secret&map=m".into(), kind: "wfs".into(), expanded: true, children: vec![rustgis_map::ServiceChild { layer, collection: "roads".into(), - source_url: "https://gis.example/roads?sig=x&sv=1".into(), + source_url: "https://gis.example/roads?sig=x-secret&sv=1".into(), }], }); let path = std::env::temp_dir().join(format!("rustgis-token-audit-{}.rgis", std::process::id())); let result = s.run("project.saveAs", &json!({"path":path})).unwrap(); let text = std::fs::read_to_string(&path).unwrap(); - for secret in ["s3cret", "pw@", "tok2", "k3y", "sig=x", "l3g"] { + for secret in ["s3cret-tile", "pw-secret@", "tok2-secret", "k3y-secret", "sig=x-secret", "l3g-secret"] { assert!(!text.contains(secret), "{secret} leaked into the file"); } assert!(text.contains("MapServer/tile/{z}/{y}/{x}") && text.contains("map=m") && text.contains("REQUEST=GetLegendGraphic")); assert!(result["stripped"].as_u64().unwrap() >= 4); assert!(s.notices.iter().any(|n| n.text.contains("saved without their keys"))); assert!(s.map().tiles.iter().any(|t| t.url == url)); - for kind in ["map", "projectPackage", "mapPackage", "mobilePackage"] { - let (_, bytes) = crate::cmd::items::item_bytes(&mut s, kind, &Value::Null).unwrap(); - let packaged = - if kind == "map" { String::from_utf8(bytes.clone()).unwrap() } else { crate::cmd::items::package_text(&bytes).unwrap() }; - for secret in ["s3cret", "pw@", "tok2", "k3y", "sig=x", "l3g"] { - assert!(!packaged.contains(secret), "{kind} leaked {secret}"); + // Every kind of item file, each with something in the project to write. + s.run("style.new", &json!({"layer": "Cities", "name": "Cities"})).unwrap(); + s.run("gp.run", &json!({"tool": "buffer", "params": {"input": "Cities", "distance": 100}})).unwrap(); + s.run("task.new", &Value::Null).unwrap(); + s.run("model.new", &json!({"fromHistory": true})).unwrap(); + s.run("report.new", &json!({"layer": "Cities"})).unwrap(); + s.run("presentation.new", &Value::Null).unwrap(); + s.run("notebook.new", &Value::Null).unwrap(); + // Kinds that hold no layer addresses, or need more than the sample gives. + let exempt = [ + ("ramps", "colour ramps hold no addresses"), + ("dlPackage", "a trained model holds no addresses"), + ("hostedImagery", "tiles rendered from a raster layer: pictures, not addresses"), + ("linkChart", "needs relationships the sample lacks"), + ]; + let mut report = Vec::new(); + for &(kind, _, _) in crate::cmd::items::ITEMS { + if exempt.iter().any(|(k, _)| *k == kind) { + continue; + } + let bytes = match crate::cmd::items::item_bytes(&mut s, kind, &Value::Null) { + Ok((_, bytes)) => bytes, + Err(e) => { + report.push(format!("{kind}: {e}")); + continue; + } + }; + let packaged = if bytes.starts_with(b"PK") { + rustgis_data::zip::read_matching(&bytes, |_| true) + .unwrap() + .into_iter() + .map(|(_, b)| String::from_utf8_lossy(&b).into_owned()) + .collect::() + } else { + String::from_utf8_lossy(&bytes).into_owned() + }; + for secret in ["s3cret-tile", "pw-secret@", "tok2-secret", "k3y-secret", "sig=x-secret", "l3g-secret"] { + if packaged.contains(secret) { + report.push(format!("{kind}: leaked {secret}")); + } } if kind == "projectPackage" { // Even a hostile archive carrying the exact registered JSON must not restore it. @@ -16121,21 +16160,22 @@ fn saved_projects_keep_no_layer_tokens() { std::fs::write(&package_path, hostile).unwrap(); let mut opened = Session::new(Project::default()); opened.run("project.open", &json!({"path":package_path})).unwrap(); - assert!(!serde_json::to_string(&opened.project).unwrap().contains("s3cret")); + assert!(!serde_json::to_string(&opened.project).unwrap().contains("s3cret-tile")); opened.run("project.openText", &json!({"text":text,"package":true})).unwrap(); - assert!(!serde_json::to_string(&opened.project).unwrap().contains("s3cret")); + assert!(!serde_json::to_string(&opened.project).unwrap().contains("s3cret-tile")); std::fs::remove_file(package_path).unwrap(); } } + assert!(report.is_empty(), "{report:#?}"); let mut t = Session::new(Project::default()); t.run("project.open", &json!({"path":path})).unwrap(); assert!(t.map().tiles.iter().any(|t| t.url == url)); - assert!(t.map().layers[0].dataset.source.as_deref().unwrap().contains("tok2")); + assert!(t.map().layers[0].dataset.source.as_deref().unwrap().contains("tok2-secret")); let modified = text.replace("\"name\":\"Secured\"", "\"name\":\"Secures\""); assert_ne!(text, modified); t.run("project.openText", &json!({"text":modified})).unwrap(); let live = serde_json::to_string(&t.project).unwrap(); - for secret in ["s3cret", "pw@", "tok2", "k3y", "sig=x"] { + for secret in ["s3cret-tile", "pw-secret@", "tok2-secret", "k3y-secret", "sig=x-secret"] { assert!(!live.contains(secret), "{secret} restored into a modified hostile file"); } let public = url.split('?').next().unwrap(); @@ -16143,7 +16183,7 @@ fn saved_projects_keep_no_layer_tokens() { hand.run("map.addTiles", &json!({"url":public,"name":"Hand made"})).unwrap(); t.run("project.openText", &json!({"text":serde_json::to_string(&hand.project).unwrap()})).unwrap(); assert!(t.map().tiles.iter().any(|t| t.url == public)); - assert!(!serde_json::to_string(&t.project).unwrap().contains("s3cret")); + assert!(!serde_json::to_string(&t.project).unwrap().contains("s3cret-tile")); std::fs::remove_file(path).unwrap(); } @@ -18810,6 +18850,11 @@ fn a_run_is_copied_from_the_history_as_code() { assert_eq!((tool_of(&runs[0]).as_deref(), tool_of(&runs[1]), tool_of(&runs[2]).as_deref()), (Some("buffer"), None, Some("buffer"))); let py = code(&mut s, 2, "python").unwrap(); assert!(py.contains("tool=\"buffer\"") && py.contains("https://h.example/a") && !py.contains("s3cr3t"), "{py}"); + // Replay keeps opaque text and unknown nested content, with an honest warning. The log's + // separate, stricter projection still exercises the legacy connection/text cases below. + assert!(py.contains("odbcpw") && py.contains("libpqpw") && py.contains("shortk"), "{py}"); + assert!(s.notices.iter().any(|n| n.text.contains("opaque parameters"))); + let py = serde_json::to_string_pretty(&crate::cmd::history::redacted(&old.params)).unwrap(); // A libpq connection keeps its words but its password, though spaces part them. assert!(py.contains("host=db") && !py.contains("hunter2"), "{py}"); // A connection with no prefix `redact` knows keeps its words but its secret-named values. @@ -18910,6 +18955,9 @@ fn secrets_in_a_query_or_a_run_s_error_stay_out_of_copied_code() { outputs: Vec::new(), }); let py = s.run("gp.historyCode", &json!({"run": 0})).unwrap()["code"].as_str().unwrap().to_string(); + assert!(py.contains("password=qp"), "opaque query text stays replayable: {py}"); + assert!(s.notices.iter().any(|n| n.text.contains("opaque parameters"))); + let py = serde_json::to_string_pretty(&crate::cmd::history::redacted(&s.project.gp_history[0].params)).unwrap(); assert!(py.contains("a=1&password=&b=2") && !py.contains("qp"), "{py}"); // An `&` in a value, or in quotes, isn't another pair: the whole value goes. So does an ODBC // braced value with an escaped `}}` in it. @@ -22923,6 +22971,41 @@ fn geopackage_trigger_rebuild_rejects_a_valid_envelope_with_unreadable_wkb() { assert_eq!(s.run("gpkg.pendingEdits", &json!({"layer":"R"})).unwrap()["updates"], 1); } +/// A template keeps no copies of the data: offline snapshots, archived edits and a streamed +/// layer's file would each bring features back when it opens. +#[test] +fn project_template_carries_no_data() { + let path = grid_fgb("template-streamed", 20); + let mut s = with_streamed(&path); + s.run("offline.download", &json!({"name": "Paris", "extent": [1.0, 48.0, 4.0, 50.0]})).unwrap(); + s.run("data.archive", &json!({"layer": "Cities"})).unwrap(); + s.run("edit.setValue", &json!({"layer": "Cities", "row": 0, "field": "name", "value": "Renamed"})).unwrap(); + s.run("graphics.add", &json!({"shape": "rectangle", "points": [[0, 45], [10, 55]]})).unwrap(); + let mut table = rustgis_data::Dataset::new( + "Visits", + rustgis_data::GeometryType::None, + vec![rustgis_data::Field::new("n", rustgis_data::FieldType::Integer)], + ); + table.push(None, vec![rustgis_data::Value::Int(1)]); + table.source = Some(path.display().to_string()); + s.project.add_table(table); + assert!(!s.project.offline_areas.is_empty()); + let (_, bytes) = crate::cmd::items::item_bytes(&mut s, "template", &Value::Null).unwrap(); + let text = String::from_utf8(bytes).unwrap(); + assert!(!text.contains(path.to_str().unwrap()), "the streamed layer's file is in the template"); + let t: Project = serde_json::from_str(&text).unwrap(); + assert!(t.offline_areas.is_empty()); + assert!(t.tables.iter().all(|t| t.dataset.features.is_empty() && t.dataset.source.is_none())); + assert!(t.maps.iter().flat_map(|m| &m.layers).all(|l| l.dataset.features.is_empty()), "graphics and features stay out"); + let cities = t.maps[0].layer_by_name("Cities").unwrap(); + assert_eq!(cities.design.archive.as_ref().map(Vec::len), Some(0), "archiving stays on, without its edits"); + let streamed = t.maps[0].layer_by_name("Streamed").unwrap(); + assert!(streamed.source.is_none() && streamed.dataset.features.is_empty()); + let mut opened = Session::new(Project::default()); + opened.run("project.fromTemplate", &json!({"text": text, "name": "Setup"})).unwrap(); + assert!(opened.project.maps.iter().flat_map(|m| &m.layers).all(|l| l.dataset.features.is_empty())); +} + #[cfg(not(target_arch = "wasm32"))] #[test] fn geopackage_template_drops_edit_binding() { diff --git a/crates/engine/src/tests/credential_output.rs b/crates/engine/src/tests/credential_output.rs new file mode 100644 index 000000000..d9d87d7a3 --- /dev/null +++ b/crates/engine/src/tests/credential_output.rs @@ -0,0 +1,315 @@ +use super::*; +use crate::cmd::notebook::{Cell, CellKind, Notebook}; +use crate::{ + GpRecord, Peer, durable, + project::{Model, ModelGraph, ModelNode, ModelStep, Task, TaskStep}, +}; +use rustgis_map::presentation::{Page, PageKind, Presentation}; + +const SECRET: &str = "RGSYNTH_output_20261103"; +const DATA: &str = "RGSYNTH_data_20261103"; + +/// Inline synthetic state exercises owning fields without contacting any service. +fn marked() -> Session { + let mut s = world(); + let url = format!("https://h.invalid/a?token={SECRET}&layer=a%20b"); + s.project.maps[0].layers[0].dataset.source = Some(url.clone()); + s.project.tasks.push(Task { name: "Credentials".into(), description: DATA.into(), steps: vec![ + TaskStep { title: DATA.into(), instructions: DATA.into(), command: "layer.addUrl".into(), params: json!({"url": url}) }, + TaskStep { title: "Malformed wrapper".into(), instructions: String::new(), command: "layer.addUrl".into(), params: json!({"url": format!("rustgis-s3:broken:{SECRET}")}) }, + TaskStep { title: "Data".into(), instructions: String::new(), command: "layer.addDataText".into(), params: json!({"text": format!(r#"{{"type":"Feature","geometry":null,"properties":{{"token":"{DATA}","key":"{DATA}","password":"{DATA}","url":"https://h.invalid/?token={DATA}"}}}}"#)}) }, + ] }); + let params = json!({"input": "Cities", "token": SECRET, "values": {"token": DATA, "key": DATA, "password": DATA}}); + s.project.models.push(Model { + name: "Credentials".into(), + steps: vec![ModelStep { tool: "buffer".into(), params: params.clone() }], + graph: Some(ModelGraph { + nodes: vec![ModelNode { id: "tool1".into(), tool: Some("buffer".into()), params: params.clone(), ..Default::default() }], + edges: Vec::new(), + }), + }); + s.project.gp_history.push(GpRecord { + tool: "Buffer".into(), + id: "buffer".into(), + params, + command: Some(("gp.bufferPoint".into(), json!({"token": SECRET}))), + at: 0, + ok: true, + messages: vec![format!("source {url}")], + millis: 0, + outputs: vec!["Cities".into()], + }); + s.project.notebooks.push(Notebook { + name: "Credentials".into(), + jupyter: None, + cells: vec![Cell { + kind: CellKind::Code, + source: format!("# {DATA}\nlayer.addUrl {{\"url\":\"{url}\"}}\nlayer.addUrl {{\"url\":\"{url}\"\n"), + output: Some(format!("source {url}")), + error: true, + jupyter: None, + }], + }); + s.project.presentations.push(Presentation { + name: "Credentials".into(), + pages: vec![Page { + title: DATA.into(), + kind: PageKind::Video { source: url, poster: None, caption: DATA.into() }, + notes: DATA.into(), + seconds: 5.0, + }], + ..Default::default() + }); + s +} + +#[test] +fn durable_writers_clean_all_owned_roles_and_preserve_the_live_project() { + let mut s = marked(); + let before = serde_json::to_value(&s.project).unwrap(); + let (text, report) = s.project.saved_text(None).unwrap(); + assert!(!text.contains(SECRET)); + assert!(report.layer_addresses() > 0 && report.steps > 0 && report.lines > 0 && report.videos > 0 && report.withheld > 0); + assert!(text.contains(DATA)); + assert_eq!(before, serde_json::to_value(&s.project).unwrap()); + let opened = Project::from_json(&text).unwrap(); + assert_eq!(opened.tasks[0].steps, s.project.tasks[0].steps); + assert_eq!(opened.models[0], s.project.models[0]); + assert_eq!(opened.notebooks[0].cells[0].source, s.project.notebooks[0].cells[0].source); + assert!(!opened.notebooks[0].cells[0].output.as_ref().unwrap().contains(SECRET), "generated text is never restored"); + assert!(!s.project.clone().recovery_text().unwrap().contains(SECRET)); + for kind in [ + "map", + "layer", + "template", + "projectPackage", + "mapPackage", + "mobilePackage", + "gpPackage", + "task", + "notebook", + "jupyter", + "presentation", + "webTool", + ] { + let (_, bytes) = crate::cmd::items::item_bytes(&mut s, kind, &Value::Null).unwrap(); + let text = if bytes.starts_with(b"PK") { + rustgis_data::zip::read_matching(&bytes, |_| true) + .unwrap() + .into_iter() + .map(|(_, b)| String::from_utf8_lossy(&b).into_owned()) + .collect::() + } else { + String::from_utf8_lossy(&bytes).into_owned() + }; + assert!(!text.contains(SECRET), "{kind}"); + assert_eq!(before, serde_json::to_value(&s.project).unwrap(), "{kind} mutated live fields"); + } + let code = s.run("gp.historyCode", &json!({"run": 0, "as": "command"})).unwrap(); + assert!(!code.to_string().contains(SECRET)); + s.project.gp_history[0].command = None; + let code = s.run("gp.historyScript", &Value::Null).unwrap(); + assert!(!code.to_string().contains(SECRET)); + assert!(code.to_string().contains(DATA), "replayable data properties must remain"); +} + +#[test] +fn durable_clean_and_opaque_content_keep_their_meaning() { + let mut s = marked(); + let original = s.project.tasks[0].steps[2].params.clone(); + let (clean, report) = crate::roles::command("layer.addDataText", &original, 0); + assert!(clean.is_none() && report.notices().is_empty()); + let source = format!("print('https://h.invalid/?token={SECRET}')"); + let notebook = Notebook { + name: "Python".into(), + jupyter: Some(std::sync::Arc::new(json!({"metadata":{"kernelspec":{"language":"python"}}}))), + cells: vec![Cell { kind: CellKind::Code, source: source.clone(), output: None, error: false, jupyter: None }], + }; + let (text, report) = durable::output_owned(notebook, |n| serde_json::to_string(n).map_err(|e| e.to_string())).unwrap(); + assert!(text.contains(SECRET) && report.notices().is_empty()); + let params = json!({"nested": {"token": SECRET}}); + let (clean, report) = crate::roles::command("plugin.unknown", ¶ms, 0); + assert!(clean.is_none() && report.unchecked == 1); + let clean = Project::default(); + let expected = serde_json::to_string(&clean).unwrap(); + let (text, report) = durable::output_owned(clean, |p| serde_json::to_string(p).map_err(|e| e.to_string())).unwrap(); + assert_eq!(text, expected); + assert!(report.notices().is_empty()); + s.project.tasks[0].steps[0].params = json!({"url": "rustgis-s3:broken:RGSYNTH_output_20261103"}); + let (text, report) = s.project.saved_text(None).unwrap(); + assert!(!text.contains(SECRET) && report.withheld > 0); +} + +#[test] +fn durable_formatter_errors_unwinds_and_apply_failures_restore_whole_fields() { + use durable::{Durable, FieldMut, Leaf, Owner, Prepare}; + let mut s = marked(); + s.project.maps[0].layers[0].dataset.source = Some("/tmp/rustgis-output/data.geojson".into()); + let mut document = Project::default(); + document.tasks = s.project.tasks.clone(); + let text = serde_json::to_string_pretty(&document).unwrap(); + let mut params = json!({"text": text}); + let mut id = "project.openText"; + for _ in 0..6 { + params = json!({"steps": [{"title":"nested", "command":id, "params":params}]}); + id = "task.new"; + } + s.project.tasks[0].steps[0].command = id.into(); + s.project.tasks[0].steps[0].params = params; + // A separate document is actually parsed and rewritten before the deeper envelope withholds. + s.project.tasks[0].steps.push(TaskStep { + title: "document".into(), + instructions: String::new(), + command: "project.openText".into(), + params: json!({"text":text}), + }); + let before = serde_json::to_value(&s.project).unwrap(); + let result = durable::output(&mut s.project, Some("/tmp/rustgis-output"), |p| { + assert_eq!(p.maps[0].layers[0].dataset.source.as_deref(), Some("./data.geojson")); + assert!(!serde_json::to_string(p).unwrap().contains(SECRET)); + Err::<(), _>("formatter failed".into()) + }); + assert!(result.is_err()); + assert_eq!(before, serde_json::to_value(&s.project).unwrap()); + #[cfg(not(target_arch = "wasm32"))] + { + let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = durable::output(&mut s.project, Some("/tmp/rustgis-output"), |_| -> Result<(), String> { + std::panic::resume_unwind(Box::new("formatter failed")) + }); + })); + assert!(unwound.is_err()); + assert_eq!(before, serde_json::to_value(&s.project).unwrap()); + } + struct Bad(Project); + impl Durable for Bad { + /// Offers classified whole fields once, before any live mutation. + fn prepare(&self, p: &mut Prepare<'_>) { + self.0.prepare(p); + p.string(Leaf::Video(usize::MAX), "https://h.invalid/?token=RGSYNTH_output_20261103", true, false); + } + + /// Resolves a prepared location through owned fields, without allocation or policy. + fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option> { + self.0.field_mut(owners, leaf) + } + } + let mut bad = Bad(s.project); + assert!(durable::output(&mut bad, Some("/tmp/rustgis-output"), |_| Ok(())).is_err()); + assert_eq!(before, serde_json::to_value(&bad.0).unwrap()); +} + +#[test] +fn model_and_presentation_non_item_writers_use_public_forms() { + let mut s = marked(); + let graph = s.project.models[0].graph.clone().unwrap(); + let out = s.run("model.export", &json!({"model":"Credentials"})).unwrap(); + assert!(!out["text"].as_str().unwrap().contains(SECRET)); + let file = std::env::temp_dir().join(format!("rustgis-output-model-{}.json", std::process::id())); + s.run("model.export", &json!({"model":"Credentials", "path":file})).unwrap(); + assert!(!std::fs::read_to_string(&file).unwrap().contains(SECRET)); + std::fs::remove_file(file).unwrap(); + assert_eq!(s.project.models[0].graph.as_ref(), Some(&graph)); + let userinfo = format!("https://u:{SECRET}@h.invalid"); + assert_eq!(crate::cmd::presentation::video_label(&userinfo), "h.invalid"); + assert_eq!(crate::cmd::presentation::video_label("/tmp/movie?cut#1.mp4"), "movie?cut#1.mp4"); + s.run("presentation.videoPage", &json!({"source":userinfo})).unwrap(); + assert_eq!(s.project.presentations[0].pages.last().unwrap().title, "h.invalid"); + let (bytes, _) = crate::cmd::presentation::export_bytes(&mut s, 0, "html", 48.0).unwrap(); + let html = String::from_utf8(bytes).unwrap(); + assert!(!html.contains(SECRET)); + assert!(html.contains(DATA)); + assert!(s.notices.iter().any(|n| n.text.contains("video address(es)"))); +} + +#[test] +fn generated_notebook_errors_are_cleaned_before_the_output_is_retained() { + let mut s = marked(); + let source = format!("layer.addUrl {{\"url\":\"https://h.invalid/?token={SECRET}\",\"cellSize\":1}}"); + s.project.notebooks[0].cells[0].source = source.clone(); + s.trusted_notebooks.insert("credentials".into()); + let _ = s.run("notebook.run", &json!({"notebook":0})); + assert_eq!(s.project.notebooks[0].cells[0].source, source); + assert!(!s.project.notebooks[0].cells[0].output.as_ref().unwrap().contains(SECRET)); + for kind in ["notebook", "jupyter"] { + assert!(!String::from_utf8_lossy(&crate::cmd::items::item_bytes(&mut s, kind, &Value::Null).unwrap().1).contains(SECRET)); + } + // Failed file tools retain their failure history through the command rollback. The missing + // input is a layer name, so this reaches the production error path without any network I/O. + let history = s.project.gp_history.len(); + assert!( + s.run( + "gp.run", + &json!({"tool":"vectorToPmtiles", "params":{"input":format!("https://h.invalid/?token={SECRET}"), "minZoom":0, "maxZoom":0}}) + ) + .is_err() + ); + assert_eq!(s.project.gp_history.len(), history + 1); + let record = s.project.gp_history.last().unwrap(); + assert_eq!(record.id, "vectorToPmtiles"); + assert!(!record.ok && !record.messages.join("\n").contains(SECRET)); +} + +#[test] +fn peers_bypass_saved_restore_and_retention_requires_the_current_exact_slot() { + let mut s = marked(); + let saved = s.project.saved_text(None).unwrap().0; + s.trusted_tasks.insert("credentials".into()); + let generation = s.opened; + s.open_peer_text(Peer::Embedder, &saved).unwrap(); + assert!(s.opened != generation && s.trusted_tasks.is_empty()); + assert!(!serde_json::to_string(&s.project).unwrap().contains(SECRET)); + for scenario in ["unchanged", "rotation", "removed", "inserted", "moved", "accounts", "generation", "duplicate_table"] { + let mut s = marked(); + s.project.tasks.clear(); + s.project.models.clear(); + s.project.gp_history.clear(); + s.project.notebooks.clear(); + s.project.presentations.clear(); + if scenario == "accounts" { + let mut other = s.project.maps[0].layers[0].clone(); + other.id = 9000; + other.dataset.source = Some("https://h.invalid/a?token=RGSYNTH_other_20261103&layer=a%20b".to_string()); + s.project.maps[0].layers.push(other); + } + if scenario == "duplicate_table" { + let mut table = rustgis_data::Dataset::new("Table", rustgis_data::GeometryType::None, vec![]); + table.source = Some(format!("https://h.invalid/table?token={SECRET}")); + s.project.add_table(table); + } + let (public, _) = s.peer_text(Peer::Bridge).unwrap(); + assert!(!public.contains(SECRET)); + let mut incoming: Value = serde_json::from_str(&public).unwrap(); + match scenario { + "rotation" => { + s.project.maps[0].layers[0].dataset.source = Some("https://h.invalid/a?token=RGSYNTH_rotated_20261103&layer=a%20b".into()) + } + "removed" => { + s.project.maps[0].layers.remove(0); + } + "inserted" => incoming["maps"][0]["layers"][0]["id"] = json!(9001), + "moved" => { + incoming["maps"][0]["name"] = json!("Another map"); + } + "generation" => { + s.run("project.new", &Value::Null).unwrap(); + } + "duplicate_table" => { + let table = incoming["tables"][0].clone(); + incoming["tables"].as_array_mut().unwrap().push(table); + } + _ => {} + } + s.open_peer_text(Peer::Bridge, &incoming.to_string()).unwrap(); + let full = serde_json::to_string(&s.project).unwrap(); + if scenario == "unchanged" || scenario == "duplicate_table" { + assert!(full.contains(SECRET), "{scenario}: ordinary layer stays"); + } else { + assert!(!full.contains(SECRET), "{scenario}"); + } + if scenario == "duplicate_table" { + assert!(s.project.tables.iter().all(|t| !t.source.as_deref().unwrap_or("").contains(SECRET))); + } + assert!(s.trusted_tasks.is_empty() && s.trusted_models.is_empty() && s.trusted_notebooks.is_empty()); + } +} diff --git a/crates/map/src/lib.rs b/crates/map/src/lib.rs index a2d4a1912..6399ca49c 100644 --- a/crates/map/src/lib.rs +++ b/crates/map/src/lib.rs @@ -22,7 +22,9 @@ pub mod projection; pub mod qgis; pub mod raster; pub mod report; +pub mod slots; pub mod source; +pub use slots::{LayerRole, Slot, SlotAt, SlotPos, TileRole}; pub mod symbol_by; pub mod symbology; pub mod tiles; @@ -2693,6 +2695,210 @@ fn de_reference_scale<'de, D: serde::Deserializer<'de>>(d: D) -> Result Vec> { + let Map { + name: _, + layers, + service_groups, + groups: _, + basemap: _, + tiles, + camera: _, + bookmarks: _, + graticule: _, + projection: _, + popup_mode: _, + label_display: _, + label_lock: _, + label_lock_line_anchors: _, + editing: _, + swipe: _, + view, + thumbnail: _, + animation: _, + time: _, + body: _, + next_layer_id: _, + instance_id: _, + } = self; + let mut out = Vec::new(); + for (group, g) in service_groups.iter().enumerate() { + let ServiceGroup { id: _, name: _, url, kind: _, expanded: _, children } = g; + out.push(Slot { at: SlotPos::Group { group, child: None }, value: url, address: true, local_path: false }); + for (child, c) in children.iter().enumerate() { + let ServiceChild { layer: _, collection: _, source_url } = c; + out.push(Slot { at: SlotPos::Group { group, child: Some(child) }, value: source_url, address: true, local_path: false }); + } + } + for (tile, t) in tiles.iter().enumerate() { + out.extend(t.output_slots().into_iter().map(|(role, value, local_path)| Slot { + at: SlotPos::Tile { tile, role }, + value, + address: true, + local_path, + })); + } + let view::ViewSettings { + effects: _, + color_vision: _, + wireframe: _, + clip_distance: _, + terrain, + fast: _, + navigator: _, + first_person: _, + } = view; + if let Some(view::Terrain { source, exaggeration: _ }) = terrain { + match source { + view::TerrainSource::Tiles { url, encoding: _ } => { + out.push(Slot { at: SlotPos::Terrain, value: url, address: true, local_path: false }) + } + view::TerrainSource::Layer(_) => {} + } + } + for (layer, l) in layers.iter().enumerate() { + out.extend(l.output_slots().into_iter().map(|(role, value, address, local_path)| Slot { + at: SlotPos::Layer { layer, role }, + value, + address, + local_path, + })); + } + out + } + + /// Resolves only the recorded location; never rediscovers or reclassifies a written value. + pub fn slot_mut(&mut self, at: &SlotPos) -> Option<&mut String> { + match at { + SlotPos::Group { group, child } => { + let g = self.service_groups.get_mut(*group)?; + match child { + Some(c) => Some(&mut g.children.get_mut(*c)?.source_url), + None => Some(&mut g.url), + } + } + SlotPos::Tile { tile, role } => self.tiles.get_mut(*tile)?.output_slot_mut(*role), + SlotPos::Terrain => match &mut self.view.terrain.as_mut()?.source { + view::TerrainSource::Tiles { url, encoding: _ } => Some(url), + view::TerrainSource::Layer(_) => None, + }, + SlotPos::Layer { layer, role } => self.layers.get_mut(*layer)?.output_slot_mut(*role), + } + } +} + +impl Layer { + /// Classifies whole owning fields; a source serving as provenance and a path occurs once. + pub fn output_slots(&self) -> Vec<(LayerRole, &str, bool, bool)> { + let Layer { + id: _, + name: _, + visible: _, + dataset, + renderer: _, + labels: _, + transparency: _, + min_scale: _, + max_scale: _, + definition_query: _, + time: _, + selection: _, + expanded: _, + selectable: _, + graphics: _, + blend: _, + mask: _, + display_filters: _, + aggregation: _, + extrusion: _, + line_strokes: _, + fill_layers: _, + no_fill: _, + stroke_levels: _, + marker_rotation: _, + symbol_by: _, + reference_scale: _, + priority: _, + design: _, + raster, + source, + postgis: _, + gpkg, + instance_id: _, + projected: _, + display: _, + display_rejected: _, + display_levels: _, + projected_extents: _, + projected_vertices: _, + thinned: _, + projected_rev: _, + projected_for: _, + view_memory: _, + geometry_kinds: _, + geometry_rev: _, + } = self; + let mut out = Vec::new(); + let rustgis_data::Dataset { + name: _, + geometry_type: _, + fields: _, + features: _, + source: provenance, + crs: _, + crs_incomplete: _, + zm_dropped: _, + windowed: _, + } = dataset; + if let Some(v) = provenance { + out.push((LayerRole::Dataset, v.as_str(), true, paths::is_local(v))); + } + if let Some(s) = source { + out.extend(s.output_slots()); + } + if let Some(r) = raster { + out.extend(r.output_slots()); + } + if let Some(g) = gpkg { + out.push((LayerRole::GpkgPath, g.path.as_str(), false, paths::is_local(&g.path))); + } + out + } + + /// Allocation-free resolution, valid while the layer's structure is held unchanged. + pub fn output_slot_mut(&mut self, role: LayerRole) -> Option<&mut String> { + match role { + LayerRole::Dataset => self.dataset.source.as_mut(), + LayerRole::Streamed | LayerRole::GdalPath | LayerRole::DuckDbDatabase => self.source.as_mut()?.output_slot_mut(role), + LayerRole::Raster | LayerRole::ZarrStore => self.raster.as_mut()?.output_slot_mut(role), + LayerRole::GpkgPath => Some(&mut self.gpkg.as_mut()?.path), + } + } +} + +impl Map { + /// Bridge identity is independent of the address value; duplicate ids remain ambiguous. + pub fn slot_identity(&self, at: &SlotPos) -> Option { + match at { + SlotPos::Group { group, child } => { + let g = self.service_groups.get(*group)?; + Some(SlotAt::Group { + id: g.id, + child: match child { + Some(i) => Some(g.children.get(*i)?.layer), + None => None, + }, + }) + } + SlotPos::Tile { tile, role } => Some(SlotAt::Tile { id: self.tiles.get(*tile)?.id.clone(), role: *role }), + SlotPos::Terrain => Some(SlotAt::Terrain), + SlotPos::Layer { layer, role } => Some(SlotAt::Layer { id: self.layers.get(*layer)?.id, role: *role }), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index ae357e7b1..dde6cc2b5 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -1079,6 +1079,43 @@ impl RasterLayer { } } +impl RasterLayer { + /// Only owning address/path strings are offered; embedded bytes and render state stay opaque. + pub(crate) fn output_slots(&self) -> Vec<(crate::LayerRole, &str, bool, bool)> { + let RasterLayer { + path, + file_name: _, + embedded, + subdataset: _, + crs: _, + time_zarr, + placement: _, + style: _, + data: _, + error: _, + masked: _, + opening: _, + } = self; + let mut out = Vec::new(); + if let Some(p) = path { + out.push((crate::LayerRole::Raster, p.as_str(), true, embedded.is_none() && crate::paths::is_local(p))); + } + if let Some(ZarrTime { store, variable: _, index: _, dim: _, offset: _, dates: _, crs: _, shown: _ }) = time_zarr { + out.push((crate::LayerRole::ZarrStore, store.as_str(), true, crate::paths::is_local(store))); + } + out + } + + /// Looks up a prepared address/path field without allocation. + pub(crate) fn output_slot_mut(&mut self, role: crate::LayerRole) -> Option<&mut String> { + match role { + crate::LayerRole::Raster => self.path.as_mut(), + crate::LayerRole::ZarrStore => Some(&mut self.time_zarr.as_mut()?.store), + _ => None, + } + } +} + #[cfg(test)] mod tests { use rustgis_data::raster::{Crs, GeoTransform}; diff --git a/crates/map/src/slots.rs b/crates/map/src/slots.rs new file mode 100644 index 000000000..48219ab17 --- /dev/null +++ b/crates/map/src/slots.rs @@ -0,0 +1,46 @@ +//! Stable leaf locations for a synchronous output transaction. Resolving a location neither +//! discovers slots nor changes structure; addresses and local paths share a single location. + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum SlotPos { + Group { group: usize, child: Option }, + Tile { tile: usize, role: TileRole }, + Terrain, + Layer { layer: usize, role: LayerRole }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LayerRole { + Dataset, + Streamed, + Raster, + ZarrStore, + GdalPath, + DuckDbDatabase, + GpkgPath, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum TileRole { + Url, + TimeUrl, + TimeAt(usize), + Legend(usize), +} + +/// A slot's identity across a bridge exchange. Positions are deliberately absent, so reordering +/// layers preserves identity while changing their owner, role or id does not. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum SlotAt { + Layer { id: u32, role: LayerRole }, + Tile { id: String, role: TileRole }, + Group { id: u32, child: Option }, + Terrain, +} + +pub struct Slot<'a> { + pub at: SlotPos, + pub value: &'a str, + pub address: bool, + pub local_path: bool, +} diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 83223919a..ddf80fed0 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -1120,6 +1120,74 @@ pub fn generation() -> u64 { GENERATION.load(Ordering::Relaxed) } +impl SourceLayer { + /// Persisted source address and local driver paths; handles and embedded data are opaque. + pub(crate) fn output_slots(&self) -> Vec<(crate::LayerRole, &str, bool, bool)> { + let SourceLayer { + path, + file_name: _, + table: _, + shapefile_crs: _, + embedded, + handle: _, + window: _, + render_lod: _, + pending: _, + projection: _, + ready_cache: _, + byte_budget: _, + estimated_row_bytes: _, + full_read: _, + saved_selection: _, + window_fields: _, + opening: _, + error: _, + login: _, + limit: _, + reads: _, + gdal, + duckdb, + postgis, + mysql, + sqlserver, + } = self; + use crate::{LayerRole as R, paths}; + let mut out = Vec::new(); + if let Some(p) = path { + out.push(( + R::Streamed, + p.as_str(), + true, + embedded.is_none() + && gdal.is_none() + && duckdb.is_none() + && postgis.is_none() + && mysql.is_none() + && sqlserver.is_none() + && paths::is_local(p), + )); + } + if let Some(g) = gdal { + out.push((R::GdalPath, g.path.as_str(), false, paths::is_local(&g.path))); + } else if let Some(d) = duckdb + && let Some(p) = &d.database + { + out.push((R::DuckDbDatabase, p.as_str(), false, paths::is_local(p))); + } + out + } + + /// Resolves a prepared field without examining its written contents. + pub(crate) fn output_slot_mut(&mut self, role: crate::LayerRole) -> Option<&mut String> { + match role { + crate::LayerRole::Streamed => self.path.as_mut(), + crate::LayerRole::GdalPath => Some(&mut self.gdal.as_mut()?.path), + crate::LayerRole::DuckDbDatabase => self.duckdb.as_mut()?.database.as_mut(), + _ => None, + } + } +} + #[cfg(test)] mod tests { /// Reopening this same layer must invalidate populated decisions even without `reset`. diff --git a/crates/map/src/tiles.rs b/crates/map/src/tiles.rs index 1c73affea..4c8879e40 100644 --- a/crates/map/src/tiles.rs +++ b/crates/map/src/tiles.rs @@ -552,6 +552,64 @@ pub fn host_of(url: &str) -> Option<&str> { Some(host.split(':').next().unwrap_or(host)) } +impl TileLayer { + /// All persisted tile addresses, with the archive URL also classified as a local path. + pub(crate) fn output_slots(&self) -> Vec<(crate::TileRole, &str, bool)> { + let TileLayer { + name: _, + kind: _, + url, + bounds: _, + subdomains: _, + min_zoom: _, + max_zoom: _, + tile_size: _, + tms: _, + attribution: _, + visible: _, + opacity: _, + overlay: _, + over: _, + blend: _, + id: _, + layers: _, + style_layers: _, + hidden_layers: _, + time_url, + time_at: _, + time_urls, + time_hidden: _, + error: _, + feature_info: _, + legends, + expanded: _, + selected: _, + single_image: _, + } = self; + use crate::TileRole as R; + let mut out = vec![(R::Url, url.as_str(), self.archive_file().is_some())]; + if let Some(v) = time_url { + out.push((R::TimeUrl, v.as_str(), false)); + } + out.extend(time_urls.iter().enumerate().map(|(i, (_, v))| (R::TimeAt(i), v.as_str(), false))); + out.extend(legends.iter().enumerate().map(|(i, v)| { + let rustgis_data::ogc::Legend { url, min_scale: _, max_scale: _ } = v; + (R::Legend(i), url.as_str(), false) + })); + out + } + + /// Resolves a fixed location, independent of the address's current spelling. + pub(crate) fn output_slot_mut(&mut self, role: crate::TileRole) -> Option<&mut String> { + match role { + crate::TileRole::Url => Some(&mut self.url), + crate::TileRole::TimeUrl => self.time_url.as_mut(), + crate::TileRole::TimeAt(i) => Some(&mut self.time_urls.get_mut(i)?.1), + crate::TileRole::Legend(i) => Some(&mut self.legends.get_mut(i)?.url), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/tiles/src/arcgis_auth.rs b/crates/tiles/src/arcgis_auth.rs index 98c38e3f3..24795839a 100644 --- a/crates/tiles/src/arcgis_auth.rs +++ b/crates/tiles/src/arcgis_auth.rs @@ -454,7 +454,7 @@ pub fn adopt_url_token(url: &str) -> Result { .split('&') .filter(|p| { let (k, v) = p.split_once('=').unwrap_or((p, "")); - if k.eq_ignore_ascii_case("token") { + if rustgis_data::address::normalized_name(k).as_deref() == Some("token") { // A token is pasted, not form-encoded: its `+` stays a `+`. token = Some(decode_with(v, false)); false diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index ca8cfe350..b18f098d2 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -1927,7 +1927,7 @@ mod tests { let history = rustgis_data::diagnostics::History::default(); let _scope = history.enter(); let base = serve(vec![ - "HTTP/1.1 403 Forbidden\r\nWWW-Authenticate: Basic realm=\"private-realm\"\r\nRetry-After: 30\r\ncf-ray: 8a1-BOG\r\nX-Amzn-RequestId: amzn-1155\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + "HTTP/1.1 403 Forbidden\r\nWWW-Authenticate: Basic realm=\"private-realm\"\r\nRetry-After: 30\r\ncf-ray: 8a1-BOG\r\nX-Amzn-RequestId: amzn-1155\r\nX-Amz-Cf-Id: trace-1155\r\nX-Amzn-Waf-Action: block\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" .into(), ]); let url = format!("{base}/refused"); @@ -1935,7 +1935,9 @@ mod tests { let records = network_records(&history, &url); assert_eq!(records.len(), 1); let details = &records[0].details; - for kept in ["retry-after: 30", "cf-ray: 8a1-BOG", "x-amzn-requestid: amzn-1155"] { + for kept in + ["retry-after: 30", "cf-ray: 8a1-BOG", "x-amzn-requestid: amzn-1155", "x-amz-cf-id: trace-1155", "x-amzn-waf-action: block"] + { assert!(details.contains(kept), "{kept} in {details:?}"); } assert!(!details.contains("private-realm"), "{details:?}"); diff --git a/crates/ui-egui/src/assistantpane.rs b/crates/ui-egui/src/assistantpane.rs index 3f5e04f14..8de0d33b2 100644 --- a/crates/ui-egui/src/assistantpane.rs +++ b/crates/ui-egui/src/assistantpane.rs @@ -1790,6 +1790,17 @@ fn inline(ui: &Ui, text: &str, t: &Tokens, size: f32, heading: bool) -> egui::te job } +/// A peer's project cannot inherit a pending approval, provider reply or queued user action. +pub(crate) fn peer_replaced(app: &mut RustGisApp) { + if let Some(agent) = &mut app.assistant.agent { + agent.project_replaced(); + } + app.assistant.inflight = None; + app.assistant.chunks = None; + app.assistant.queued = None; + app.assistant.always = false; +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/ui-egui/src/connectiontree.rs b/crates/ui-egui/src/connectiontree.rs index 85bc15223..11e457f1a 100644 --- a/crates/ui-egui/src/connectiontree.rs +++ b/crates/ui-egui/src/connectiontree.rs @@ -787,8 +787,16 @@ mod tests { let slot = Slot::default(); *slot.lock().unwrap() = Some(Err("unreachable".to_string())); let params = json!({"url": api, "layer": "relief (map tiles)", "text": tiled, "name": "Relief"}); - let resolve = - crate::TilesResolve { params, list: "l".into(), doc: "l".into(), read: 0, vector: false, fallback: fallback.into(), slot }; + let resolve = crate::TilesResolve { + params, + list: "l".into(), + doc: "l".into(), + read: 0, + withheld_addresses: 0, + vector: false, + fallback: fallback.into(), + slot, + }; crate::ServiceStep::Tiles { resolve, dialog: false, origin: origin.map(str::to_string) } }; let before = h.state().session.sources.recent.clone(); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index ef1cbb719..efaf01ebe 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -411,11 +411,13 @@ pub struct ServiceBrowse { pub struct TilesResolve { /// `service.add`'s parameters, given the template as `"tiles"` once read. pub params: Value, - /// The tileset list, whose query the template keeps; the document being read; how many have + /// The tileset list, whose safe query options the template may inherit; the document being read; how many have /// been. pub list: String, pub doc: String, pub read: u8, + /// Destinations that lost inherited query settings across both document reads. + pub withheld_addresses: u32, pub vector: bool, /// The usual-path template, when the advertised one can't be read. pub fallback: String, @@ -1556,7 +1558,7 @@ impl RustGisApp { // Not an undoable edit: saving names the project, as `project.saveAs` does, once the // file is written. let old = std::mem::replace(&mut self.session.project.name, stem.clone()); - let (text, stripped) = match self.session.project.saved_text(None) { + let (text, report) = match self.session.project.saved_text(None) { Ok(saved) => saved, Err(e) => { self.session.project.name = old; @@ -1569,14 +1571,8 @@ impl RustGisApp { return Err(e); } self.session.notify(crate::i18n::fmt("Downloaded {name}", &[("name", &file)]), false); - if stripped > 0 { - self.session.notify( - crate::i18n::fmt( - "{n} layer address(es) were saved without their keys, tokens or passwords: after RustGIS closes, add those layers again to give them", - &[("n", &stripped.to_string())], - ), - false, - ); + for notice in report.notices() { + self.session.notify(notice, false); } self.session.mark_saved(); self.ui.web_file = Some(file); @@ -1797,6 +1793,29 @@ impl RustGisApp { self.run(id, params) } + /// Imports a peer's project without going through User command dispatch or restore memory. + pub fn open_peer_text(&mut self, text: &str) -> Result<(), String> { + let report = self.session.open_peer_text(rustgis_engine::Peer::Embedder, text)?; + assistantpane::peer_replaced(self); + self.ui.web_file = None; + self.map.deleting = None; + self.ui.cog_probes.clear(); + if let Some(steps) = self.ws.recording.take() { + self.session.notify( + crate::i18n::fmt( + "Recording stopped: another project opened, and its {n} steps were for the previous one.", + &[("n", &steps.len().to_string())], + ), + true, + ); + } + layoutview::follow_project(self); + for notice in report.notices() { + self.session.notify(notice, false); + } + Ok(()) + } + /// Runs a command by id: `ui.*` here, everything else in the engine. pub fn run(&mut self, id: &str, params: Value) -> Result { let _diagnostics = self.session.diagnostics.enter(); @@ -3773,6 +3792,10 @@ impl RustGisApp { continue; }; let next = r.ok().and_then(|body| resolve_tiles(&resolve.list, &resolve.doc, &body, resolve.vector)); + let next = next.map(|resolved| { + resolve.withheld_addresses = resolve.withheld_addresses.saturating_add(resolved.withheld_addresses); + resolved.step + }); let template = match next { Some(TilesStep::Next(next)) if resolve.read < 2 => { resolve.read += 1; @@ -3784,6 +3807,12 @@ impl RustGisApp { Some(TilesStep::Template(t)) => t, _ => resolve.fallback.clone(), }; + if resolve.withheld_addresses > 0 { + self.session.notify( + i18n::fmt(rustgis_engine::data::ogc::QUERY_WITHHELD_NOTICE, &[("n", &resolve.withheld_addresses.to_string())]), + false, + ); + } let mut p = resolve.params; p["tiles"] = json!(template); let title = p["name"].as_str().unwrap_or("").to_string(); @@ -4537,6 +4566,37 @@ mod download_service_tests { use super::{RustGisApp, Services, complete_gp_file}; use serde_json::json; + /// The real asynchronous completion receiver owns the forwarding notice, even when the + /// layer is added successfully. Input is already downloaded into the usual completion slot. + #[test] + fn tile_resolver_completion_reports_query_settings_it_withheld() { + for (query, count) in [("collections=roads", 0), ("token=RGSYNTH_ui_20261103&collections=roads", 1)] { + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + let list = format!("https://h.invalid/svc/tiles?{query}"); + let capabilities = json!({"title":"roads", "links":[{"rel":"tilesets-map", "href":list}]}).to_string(); + let body = + br#"{"links":[{"rel":"item", "type":"image/png", "href":"https://elsewhere.invalid/{tileMatrix}/{tileRow}/{tileCol}"}]}"# + .to_vec(); + let resolve = super::TilesResolve { + params: json!({"url":"https://h.invalid/svc", "type":"ogcapi", "layer":"map", "text":capabilities}), + list: list.clone(), + doc: list, + read: 0, + withheld_addresses: 0, + vector: false, + fallback: "https://h.invalid/svc/tiles/{z}/{y}/{x}".into(), + slot: std::sync::Arc::new(std::sync::Mutex::new(Some(Ok(body)))), + }; + app.ui.service_steps.push(super::ServiceStep::Tiles { resolve, dialog: true, origin: None }); + app.poll_service_steps(); + assert!(app.ui.service_steps.is_empty()); + let notices = app.session.notices.iter().filter(|n| n.text.contains("address(es) the service listed")).count(); + assert_eq!(notices, count); + assert!(app.session.map().tiles.iter().all(|t| !t.url.contains("RGSYNTH_"))); + assert!(app.dialogs.error.is_none(), "{:?}", app.dialogs.error); + } + } + #[test] fn rejected_download_does_not_announce_success_or_rename_project() { let mut app = RustGisApp::with_sample() @@ -4669,3 +4729,25 @@ mod download_service_tests { assert_eq!(app.session.notices.iter().filter(|n| n.error && n.text == "no pending file with that id").count(), 1); } } + +#[cfg(test)] +mod peer_output_tests { + use super::*; + + /// Drives the actual receiving wrapper, including the UI state a direct Session call misses. + #[test] + fn peer_project_uses_import_authority_and_resets_old_ui_work() { + let mut app = RustGisApp::with_sample(); + app.session.project.maps[0].layers[0].dataset.source = Some("https://h.invalid/?token=RGSYNTH_4".into()); + let (text, _) = app.session.project.saved_text(None).unwrap(); + app.session.trusted_tasks.insert("prior".into()); + app.assistant.queued = Some("old project action".into()); + app.ui.web_file = Some("prior.rgis".into()); + let generation = app.session.opened; + app.open_peer_text(&text).unwrap(); + assert_ne!(app.session.opened, generation); + assert!(app.session.trusted_tasks.is_empty()); + assert!(app.ui.web_file.is_none() && app.assistant.queued.is_none()); + assert!(!serde_json::to_string(&app.session.project).unwrap().contains("RGSYNTH_4")); + } +} diff --git a/crates/ui-egui/src/serviceui.rs b/crates/ui-egui/src/serviceui.rs index 2e48a4424..10c037749 100644 --- a/crates/ui-egui/src/serviceui.rs +++ b/crates/ui-egui/src/serviceui.rs @@ -119,7 +119,16 @@ pub(crate) fn add_layer_from( let fallback = offer.get("url").and_then(Value::as_str).unwrap_or("").to_string(); let vector = offer.get("type").and_then(Value::as_str) == Some("vectorTiles"); let slot = crate::fetch_into(list); - let resolve = crate::TilesResolve { params: p, list: list.to_string(), doc: list.to_string(), read: 0, vector, fallback, slot }; + let resolve = crate::TilesResolve { + params: p, + list: list.to_string(), + doc: list.to_string(), + read: 0, + withheld_addresses: 0, + vector, + fallback, + slot, + }; app.ui.service_steps.push(crate::ServiceStep::Tiles { resolve, dialog, origin }); Ok(tl!("Reading the tileset of")) } diff --git a/docs/contributing.md b/docs/contributing.md index 496098f86..d62568486 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -33,6 +33,14 @@ them together. Reply on the comment that it's left for the release quality pass reply with ``), so it can be found. Bugs, crashes, silent data loss and breaks of the rules in AGENTS.md are fixed in the pull request. +Changes to saved or generated output must classify each new owning field: address, executable +parameters, generated text or user data. Use the shared durable-output transaction and keep +credential removal separate from execution and approval identity. Review every writer that +consumes the owner, including packages and peer exchange. Reuse a focused regression that checks +the written output, restoration of live fields on failure, counted notices and clean-input +silence. Opaque data needs an explicit preservation policy; recursive removal by key name can +destroy user data. + ## Developer checks `cargo xtask ci` runs `cargo fmt --all --check`, diff --git a/docs/mcp.md b/docs/mcp.md index baf2b57b8..b12a6f3db 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -156,7 +156,11 @@ Passwords, keys and tokens in the arguments, a `run` script's lines included, ar they are written. On Linux and macOS the folder and files are readable by you only; on Windows they're protected by your user profile's permissions. A batch's calls share the batch's `duration_s` and are marked `"batch": true`. An error whose text mentions a credential (a token, a -password, a bearer…) is logged as withheld rather than risk keeping the credential. Days older than 30 are removed. +password, a bearer…) is logged as withheld rather than risk keeping the credential. Log days older +than 30 days are pruned when a logged call creates a new day's file, not by a timer; older files +can remain while logging is idle. You can remove unwanted day files from this folder yourself. +Updating does not rewrite old logs. If an older log exposed a credential, rotate it with its +service before sharing the log. ### MCP over HTTP diff --git a/docs/python.md b/docs/python.md index 7d8e96725..74af59469 100644 --- a/docs/python.md +++ b/docs/python.md @@ -139,6 +139,13 @@ The project, with the data that travels with it, goes to that app through the br (`postMessage`); nothing goes to a server. The app runs in your browser, so with the default address your data stays on your machine. Point `url=` only at a copy of the app you trust. +The widget receives public addresses and cleaned executable parameters. Credentials held by +Python are not sent to it, so protected layers can remain empty there; a warning counts affected +layer addresses. When the widget sends a project back, Python can retain credentials only for +unchanged, uniquely matched sources from its last exchange that still match its current project. +Changing or replacing a source can require signing in again in Python. Exchange notices are +available through `m.notices()` on the `Map` or `MapWidget`. + ## The app in a Qt application `rustgis.qt.MapWidget` is the same app in a Qt widget, for a desktop tool or a script that opens a @@ -287,9 +294,10 @@ last view in memory; run `layer.loadAll` on it before `to_geojson` or `to_geopan of them. Open an existing project with `Session(project="atlas.rgis")` or `s.open("atlas.rgis")`. -`s.project_json()` returns its JSON text; `s.run("project.openText", text=...)` opens project -JSON without an intermediate file. `to_png()` returns PNG bytes even when it also writes a -file; omit the path to keep the image in memory. +`s.project_json()` returns JSON with the same credential removal as Save, with notices available +through `s.notices()`. It refuses unfinished imports instead of writing runtime reservations. +`s.run("project.openText", text=...)` opens project JSON without an intermediate file. `to_png()` +returns PNG bytes even when it also writes a file; omit the path to keep the image in memory. ## From source diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index b29cc2b65..b3ecf6981 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1108,13 +1108,17 @@ routes, areas, observations…) to draw into. Add Folder connects a folder to th also lists the project's layouts, styles and tasks. A task's steps each run a command; New Task From History turns the geoprocessing history into a task you can replay. -Saved projects, map and layer files, and packages keep no password, key, token or signature in -layer web addresses. The open project keeps them for this session. RustGIS remembers the 32 most -recent distinct saved snapshots whose addresses needed stripping. Opening a plain `.rgis` file -whose bytes exactly match one of those snapshots restores its secrets; saving more distinct -snapshots evicts the oldest. Changed files, packages, imports and files saved by another process -never restore secrets. After a snapshot is evicted or RustGIS closes, add those layers again with -their credentials. +Saved projects, templates, map and layer files, and packages remove credentials from supported +addresses and recognized executable parameters. The open project keeps its credentials. Removal +notices describe what a shared workflow or video will need again; an unchecked-content notice +means opaque content was kept and needs your review. + +RustGIS remembers up to 32 distinct saved snapshots for explicit project reopening, with at most +1 MiB of restoration data per snapshot and 8 MiB in total. A snapshot must match, and only its +recorded, unchanged fields can regain credentials. Larger restoration records are not remembered; +a notice says so. Imports, packages and peer exchanges do not use this memory. After eviction or +closing RustGIS, sign in again or add the protected sources with their credentials. Feature data +and user text remain data, so review them before sharing. ### Complete streamed Shapefile rendering diff --git a/docs/user-guide/web-data.md b/docs/user-guide/web-data.md index d09d2b8d4..84f75a7fa 100644 --- a/docs/user-guide/web-data.md +++ b/docs/user-guide/web-data.md @@ -495,6 +495,14 @@ rustgis-cli run --cmd 'portal.files={"dataset":"giswqs/geospatial"}' rustgis-cli run --sample --cmd 'portal.add={"url":"https://data.geopf.fr/telechargement/download/LiDARHD-NUALID/NUALHD_1-0__LAZ_LAMB93_KE_2025-06-06/LHD_FXX_0652_6864_PTS_LAMB93_IGN69.copc.laz","name":"LiDAR HD 0652-6864"}' ``` +Service discovery forwards only a small set of query options to addresses supplied by the +service: `collections`, `datetime`, `subset`, `subset-crs`, `crs`, `bgcolor` and `transparent`. +Other settings, including keys, are inherited only by sub-addresses RustGIS constructs within +the original source path. A counted notice says when a listed address did not receive settings. +If it needs authentication, supply an explicitly authorized address for that source rather than +relying on a token being copied from the discovery address. Destination addresses keep settings +they already contain. + ### Layers that change with time A WMS or WMTS layer with a time dimension, from any service, goes on the diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index f9111a45e..fcd8b7117 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -32,9 +32,12 @@ The browser app doesn't ask yet; it warns before you leave a page with unsaved c default), RustGIS keeps your changes as a [recovery copy](#autosave-and-recovery) and offers them back the next time it starts; with autosave off, they are lost, so save before you quit. -Saved layer addresses omit passwords, access keys, tokens and signatures. The open session -keeps them, but after restarting RustGIS a protected source may need you to sign in to its -connection again or add it again with credentials. +Saved layer and video addresses, and recognized task, model, notebook and history parameters, +omit passwords, access keys, tokens and signatures. Notices count what was removed or could not +be checked. The open session keeps its credentials, but a saved workflow or protected source may +need them again when reopened. User notes, feature data and opaque content are preserved; see +[Sharing projects and generated items](https://github.com/opengeos/RustGIS/blob/main/SECURITY.md#sharing-projects-and-generated-items) +before sharing a file. ### Autosave and recovery From 1ded5bdde7797983b2851eb73c237660e3c576f6 Mon Sep 17 00:00:00 2001 From: mavrick Date: Fri, 6 Nov 2026 19:39:21 -0600 Subject: [PATCH 2/4] fix(security): preserve exact sources and counted output losses --- apps/rustgis-python/python/rustgis/_embed.py | 2 +- apps/rustgis-python/tests/test_widget.py | 8 + apps/rustgis-web/src/web.rs | 2 +- crates/assistant/src/audit.rs | 9 +- crates/assistant/src/tests.rs | 7 + crates/assistant/src/tools.rs | 112 ++++++++- crates/data/src/address.rs | 20 +- crates/data/src/ogc.rs | 31 ++- crates/engine/src/cmd/history.rs | 3 +- crates/engine/src/cmd/notebook.rs | 4 +- crates/engine/src/cmd/service.rs | 93 +++++--- crates/engine/src/cmd/service_group.rs | 43 ++-- crates/engine/src/durable.rs | 66 ++++- crates/engine/src/lib.rs | 10 +- crates/engine/src/peer.rs | 238 ++++++++++++++++--- crates/engine/src/project.rs | 47 ++-- crates/engine/src/project_log.rs | 19 ++ crates/engine/src/roles.rs | 16 +- crates/engine/src/tests/credential_output.rs | 148 +++++++++++- crates/engine/src/tests/service_logins.rs | 54 +++++ crates/map/src/source.rs | 2 +- crates/ui-egui/src/lib.rs | 62 ++++- crates/ui-egui/src/serviceui.rs | 31 ++- docs/python.md | 8 +- docs/user-guide/workspace.md | 6 +- 25 files changed, 881 insertions(+), 160 deletions(-) diff --git a/apps/rustgis-python/python/rustgis/_embed.py b/apps/rustgis-python/python/rustgis/_embed.py index c3072d757..d2c04867d 100644 --- a/apps/rustgis-python/python/rustgis/_embed.py +++ b/apps/rustgis-python/python/rustgis/_embed.py @@ -208,7 +208,7 @@ def _push(self) -> None: if text != self._project or notices != self._sent_notices: self._hold([(notice, False) for notice in notices]) self._sent_notices = notices - if keys > self._sent_keys: + if keys and (text != self._project or keys > self._sent_keys): self._hold([(f"{keys} layer address(es) went to the map widget without their keys, tokens, passwords or URL fragments; restore missing values if needed", False)]) self._sent_keys = keys except RustGISError as e: diff --git a/apps/rustgis-python/tests/test_widget.py b/apps/rustgis-python/tests/test_widget.py index 66d93945a..f8c608a88 100644 --- a/apps/rustgis-python/tests/test_widget.py +++ b/apps/rustgis-python/tests/test_widget.py @@ -120,6 +120,14 @@ def test_widget_exchange_never_sends_keys_and_checks_the_current_slot(): m.run("info.project") assert m._project == before assert m.notices() == [] + # A different protected source still has one omitted address and needs a fresh warning. + project = json.loads(m._project) + project["maps"][0]["layers"][0]["dataset"]["source"] = f"https://h.invalid/b?token={secret}" + s.run("project.openText", text=json.dumps(project)) + m.run("info.project") + assert any("without their keys" in notice for notice, _ in m.notices()) + m.run("info.project") + assert m.notices() == [] incoming = json.loads(m._project) incoming["maps"][0]["layers"][0]["name"] = "Styled by app" from_app(m, json.dumps(incoming)) diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index d920d6b31..867a6259c 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -418,7 +418,7 @@ impl Embed { app.session.notify(notice, false); } } - Err(e) => log::error!("couldn't send the project: {e}"), + Err(e) => app.session.notify(e, true), }, (false, Some(wait)) => ctx.request_repaint_after(wait), (false, None) => {} diff --git a/crates/assistant/src/audit.rs b/crates/assistant/src/audit.rs index d22b2a38a..9b93da7d5 100644 --- a/crates/assistant/src/audit.rs +++ b/crates/assistant/src/audit.rs @@ -234,7 +234,7 @@ fn redact_script(script: &str) -> String { .lines() .take(rustgis_engine::cmd::workflows::MAX_SCRIPT_LINES) .map(|line| match rustgis_engine::script_line(line) { - None => line.trim().to_string(), + None => redact_string(line.trim()), Some(Ok((id, params))) => { let id = redact_string(&id); if params.is_null() { id } else { format!("{id} {}", rustgis_engine::log_params(¶ms).into_log_value()) } @@ -329,6 +329,13 @@ mod tests { let logged = redact_script(script); assert!(logged.starts_with("map.zoomIn {}\nlayer.addUrl [parameters not JSON; withheld]\nmap.zoomOut {}")); assert!(!logged.contains("RGSYNTH_")); + let comments = "# clean comment\n# https://h.invalid/a?token=RGSYNTH_comment_20261106"; + let args = normalized_args("run", &json!({"script": comments})); + assert!(!args.to_string().contains("RGSYNTH_comment")); + assert!(!redact_script(comments).contains("RGSYNTH_comment")); + assert!(redact_script(comments).starts_with("# clean comment\n")); + let nested = json!({"key":"abc", "source":{"url":"https://h.invalid/data"}}); + assert_eq!(normalized_args("run_command", &json!({"command":"plugin.unknown", "params":nested}))["params"]["key"], ""); let document = json!({"type": "FeatureCollection", "features": [], "padding": "x".repeat(250), "token": "RGSYNTH_data_20261103"}).to_string(); let arguments = json!({"command": "layer.addDataText", "params": {"text": document}}); diff --git a/crates/assistant/src/tests.rs b/crates/assistant/src/tests.rs index 8ed0af6f8..e04703a63 100644 --- a/crates/assistant/src/tests.rs +++ b/crates/assistant/src/tests.rs @@ -4345,4 +4345,11 @@ fn peer_replacement_cancels_approvals_and_stale_replies() { assert!(agent.transcript.len() >= transcript); let _ = agent.receive(generation, ok(say("stale")), &mut s); assert!(!agent.busy() && agent.pending(&s).is_none()); + assert!(matches!(agent.ask("start a new request", &s), Step::Send(_))); + let current = agent.generation(); + let transcript = agent.transcript.len(); + let _ = agent.receive(generation, ok(say("stale while waiting")), &mut s); + assert_eq!(agent.generation(), current); + assert!(agent.waiting() && agent.pending(&s).is_none()); + assert_eq!(agent.transcript.len(), transcript); } diff --git a/crates/assistant/src/tools.rs b/crates/assistant/src/tools.rs index 359adae7b..15a20386d 100644 --- a/crates/assistant/src/tools.rs +++ b/crates/assistant/src/tools.rs @@ -279,7 +279,7 @@ pub fn run_with_budget(s: &mut Session, call: &ToolCall, tokens: usize) -> Resul } let (before, mark) = (layer_snap(s), s.notice_mark()); let r = s.run(&id, ¶ms_arg(a)?).map_err(|e| public_text(error_with_notices(e, s, mark)))?; - with_notices(with_new_layers(r, s, &before), s, mark) + return Ok(command_result(s, r, &before, mark, tokens)); } "run_tool" => { let tool = str_arg(a, "tool").ok_or("tool is required")?; @@ -287,7 +287,7 @@ pub fn run_with_budget(s: &mut Session, call: &ToolCall, tokens: usize) -> Resul let r = s .run("gp.run", &json!({"tool": tool, "params": params_arg(a)?})) .map_err(|e| public_text(error_with_notices(e, s, mark)))?; - with_notices(with_new_layers(r, s, &before), s, mark) + return Ok(command_result(s, r, &before, mark, tokens)); } "read_features" => read_features(s, a)?, // MCP's `run` (not offered to the in-app assistant): a script in one call. @@ -312,7 +312,13 @@ pub fn run_with_budget(s: &mut Session, call: &ToolCall, tokens: usize) -> Resul "test_panic" => panic!("a test tool panicked"), other => return Err(format!("unknown tool '{other}'")), }; - Ok(public_result(s, v, tokens)) + if call.name == "read_features" { + Ok(public_envelope(s, v, tokens)) + } else { + let mut withheld = Withheld::default(); + let v = public_metadata(v, 0, &mut withheld); + Ok(finish_result(s, v, tokens, withheld)) + } } /// How deep a result is searched for runtime login addresses; what is deeper is withheld. @@ -326,9 +332,51 @@ fn public_text(text: String) -> String { } /// Publishes a completed result, reporting withheld subtrees before enforcing the response budget. +#[cfg(test)] fn public_result(s: &mut Session, value: Value, tokens: usize) -> String { let mut withheld = Withheld::default(); - let mut value = public_addresses(value, 0, &mut withheld); + let value = public_addresses(value, 0, &mut withheld); + finish_result(s, value, tokens, withheld) +} + +/// These fields belong to the adapter's response schema; only their values contain data. +fn public_envelope(s: &mut Session, value: Value, tokens: usize) -> String { + let mut withheld = Withheld::default(); + let value = match value { + Value::Object(o) => Value::Object(o.into_iter().map(|(key, value)| (key, public_addresses(value, 1, &mut withheld))).collect()), + value => public_addresses(value, 0, &mut withheld), + }; + finish_result(s, value, tokens, withheld) +} + +/// Generated search, tool and project metadata has a fixed schema at every level. Its +/// strings may hold private data, but its keys cannot be login values. Row/property maps +/// and arbitrary command results use `public_addresses` instead. +fn public_metadata(value: Value, depth: usize, withheld: &mut Withheld) -> Value { + match value { + Value::String(text) => Value::String(public_text(text)), + Value::Array(a) if depth < MAX_DEPTH => Value::Array(a.into_iter().map(|v| public_metadata(v, depth + 1, withheld)).collect()), + Value::Object(o) if depth < MAX_DEPTH => { + Value::Object(o.into_iter().map(|(key, v)| (key, public_metadata(v, depth + 1, withheld))).collect()) + } + Value::Array(_) | Value::Object(_) => { + withheld.depth = withheld.depth.saturating_add(1); + json!("[too deep]") + } + other => other, + } +} + +/// Cleans the arbitrary command payload before adding adapter-owned notices/layer metadata. +fn command_result(s: &mut Session, value: Value, before: &[(String, Vec)], mark: u64, tokens: usize) -> String { + let mut withheld = Withheld::default(); + let value = public_addresses(value, 0, &mut withheld); + let value = with_notices(with_new_layers(value, s, before), s, mark); + finish_result(s, value, tokens, withheld) +} + +/// Publishes counted projection losses without sending the completed envelope through a second redactor. +fn finish_result(s: &mut Session, mut value: Value, tokens: usize, withheld: Withheld) -> String { if withheld.depth > 0 || withheld.keys > 0 { let mark = s.notice_mark(); if withheld.depth > 0 { @@ -488,7 +536,7 @@ fn with_new_layers(r: Value, s: &Session, before: &[(String, Vec)]) -> Valu let same = before.iter().find(|(name, _)| name.eq_ignore_ascii_case(&m.name)).or_else(|| if renamed { before.get(active) } else { None }); let had = same.map_or(&[][..], |(_, ids)| ids.as_slice()); - let added: Vec = m.layers.iter().filter(|l| !had.contains(&l.id)).map(|l| l.name.clone()).collect(); + let added: Vec = m.layers.iter().filter(|l| !had.contains(&l.id)).map(|l| public_text(l.name.clone())).collect(); if added.is_empty() { return if r.is_null() { json!({"ok": true}) } else { r }; } @@ -519,7 +567,7 @@ fn run_script(s: &mut Session, a: &Value, tokens: usize) -> Result { // Nothing was kept, so no line ran and no layer was added. @@ -528,7 +576,7 @@ fn run_script(s: &mut Session, a: &Value, tokens: usize) -> Result = notices.iter().filter(|t| **t != e && **t != told).collect(); let out = json!({"success": false, "result": null, "lines": 0, "layersAdded": [], "notices": said, "error": clip(&e, 8000), "duration_s": secs}); - Err(public_result(s, out, tokens)) + Err(public_envelope(s, out, tokens)) } } } @@ -536,7 +584,7 @@ fn run_script(s: &mut Session, a: &Value, tokens: usize) -> Result Value { - let said: Vec<&str> = s.notices_since(mark).iter().map(|n| n.text.as_str()).collect(); + let said: Vec = s.notices_since(mark).iter().map(|n| public_text(n.text.clone())).collect(); if said.is_empty() { return r; } @@ -901,6 +949,54 @@ mod tests { assert!(listed.contains(&plain), "the model still learns the layer's address: {listed}"); } + /// The known response schema survives a secret that happens to name one of its fields. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn run_envelope_keys_are_not_data_derived_login_values() { + if std::env::var_os("RUSTGIS_TEST_RUN_ENVELOPE").is_none() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "tools::tests::run_envelope_keys_are_not_data_derived_login_values"]) + .env("RUSTGIS_TEST_RUN_ENVELOPE", "1") + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("1 passed; 0 failed"), + "{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let mut s = Session::new(rustgis_engine::Project::default()); + let id = s + .run("connection.new", &json!({"kind":"wms", "address":"https://envelope.invalid/wms", "auth":"basic", "user":"reader"})) + .unwrap()["id"] + .clone(); + s.run("connection.login", &json!({"id":id, "password":"success"})).unwrap(); + for (script, success) in [("map.zoomIn {}", true), ("not.a.command {}", false)] { + let call = ToolCall { id: "envelope".into(), name: "run".into(), args: json!({"script":script}) }; + let text = run(&mut s, &call).unwrap_or_else(|e| e); + let out: Value = serde_json::from_str(&text).unwrap(); + assert_eq!(out.get("success"), Some(&json!(success)), "{text}"); + assert!(out.get("result").is_some() && out.get("notices").is_some() && out.get("duration_s").is_some()); + } + let data = public_addresses(json!({"success":"private field"}), 0, &mut Withheld::default()); + assert_eq!(data["[redacted]"], "private field", "the same spelling in dynamic data is still scrubbed"); + s.run("connection.login", &json!({"id":id, "password":"id"})).unwrap(); + for (name, args, array) in + [("search_commands", json!({"query":"zoom"}), "commands"), ("search_tools", json!({"query":"buffer"}), "tools")] + { + let call = ToolCall { id: "metadata".into(), name: name.into(), args }; + let text = run(&mut s, &call).unwrap(); + let out: Value = serde_json::from_str(&text).unwrap(); + let entries = out.get(array).and_then(Value::as_array).unwrap(); + assert!(!entries.is_empty(), "{text}"); + assert!(entries.iter().all(|entry| entry.get("id").is_some()), "fixed nested schema keys survive: {text}"); + } + let data = public_addresses(json!({"id":"private field"}), 0, &mut Withheld::default()); + assert_eq!(data["[redacted]"], "private field"); + } + #[test] fn wrappers_after_a_prefix_are_made_plain_too() { let origin = "https://assistant-login.example"; diff --git a/crates/data/src/address.rs b/crates/data/src/address.rs index 84ba823d7..0eba9a766 100644 --- a/crates/data/src/address.rs +++ b/crates/data/src/address.rs @@ -104,7 +104,7 @@ const SAS_KEYS: &[&str] = /// Exact and containment rules over canonical names, plus query-wide context. fn credential(k: &str, query: &QueryNames) -> bool { - matches!(k, "key" | "pwd" | "sig" | "signature" | "sas" | "authorization" | "proxyauthorization" | "cookie" | "setcookie" | "bearer") + matches!(k, "key" | "appid" | "subscriptionkey" | "clientkey" | "saskey" | "pwd" | "sig" | "signature" | "sas" | "authorization" | "proxyauthorization" | "cookie" | "setcookie" | "bearer") // These public refusal and tracing headers share the signing family's prefix. || (k.starts_with("xamz") && !matches!(k, "xamznrequestid" | "xamznwafaction" | "xamzcfid")) || k.starts_with("xgoog") @@ -527,6 +527,7 @@ mod tests { #[test] fn canonical_names_remove_encoded_and_unresolved_pairs_without_touching_values() { let over = "a".repeat(MAX_NAME_BYTES + 1); + let mut unredacted = Vec::new(); for key in [ "%74oken", "%2574oken", @@ -544,16 +545,31 @@ mod tests { "sig=M&%73v", "\\u0074oken", "accessKeyId", + "appid", + "APP_ID", + "%2561ppid", + "subscription-key", + "Subscription_Key", + "%73ubscription%2Dkey", + "client-key", + "clientKey", + "sas-key", + "SAS_Key", "%2525252574oken", over.as_str(), ] { let address = format!("https://h.invalid/a?{key}=RGSYNTH_name_20261103&layer=a%20b&f=json&%zz=1"); - let Public::Changed { address: public, removed, .. } = public_web_address(&address) else { panic!("{key}") }; + let Public::Changed { address: public, removed, .. } = public_web_address(&address) else { + unredacted.push(key.to_string()); + continue; + }; assert!(!public.contains("RGSYNTH_name_20261103"), "{key}: {public}"); assert!(public.ends_with("layer=a%20b&f=json&%zz=1"), "{public}"); assert!(removed > 0); } + assert!(unredacted.is_empty(), "credential names left unchanged: {unredacted:?}"); for query in [ + "client_id=public&application_id=public&map=1", "layer=a%20b&f=json&%zz=1", "layer=token%3DM", "code=EPSG:4326", diff --git a/crates/data/src/ogc.rs b/crates/data/src/ogc.rs index 4a3e3729a..fa7684594 100644 --- a/crates/data/src/ogc.rs +++ b/crates/data/src/ogc.rs @@ -533,7 +533,7 @@ fn is_data_link(link: &serde_json::Value) -> bool { } /// Chooses JSON before an untyped data link, without guessing an endpoint or copying credentials. -fn collections_link(address: &str, links: &[serde_json::Value]) -> Option { +fn collections_link(address: &str, links: &[serde_json::Value]) -> Option<(String, bool)> { let base = url::Url::parse(crate::login_address::inner(address)).ok()?; links .iter() @@ -555,7 +555,12 @@ fn collections_link(address: &str, links: &[serde_json::Value]) -> Option Result { let title = v.get("title").and_then(|t| t.as_str()).unwrap_or_default().to_string(); let mut service = Service { kind: "OGC API Tiles".into(), title: title.clone(), ..Service::default() }; let links = v.get("links").and_then(|l| l.as_array()).cloned().unwrap_or_default(); - service.collections_url = collections_link(url, &links); + if let Some((next, withheld)) = collections_link(url, &links) { + service.collections_url = Some(next); + service.withheld_addresses = u32::from(withheld); + } let name = if title.is_empty() { "tiles".to_string() } else { title }; // The dataset's layers aren't listed here, so its vector tiles can't be styled: map tiles only. if let Some((t, list, withheld)) = tileset(url, &links, TileRel::Map) { @@ -3992,11 +4000,12 @@ mod tests { let address = crate::login_address::bind(binding, origin, "https://maps.example.org/api?key=a%2Fb").unwrap(); for href in ["/collections", "https://MAPS.example.org:443/collections"] { let links = serde_json::json!([{"rel":"data","type":"application/json","href":href}]); - let next = collections_link(&address, links.as_array().unwrap()).unwrap(); + let (next, withheld) = collections_link(&address, links.as_array().unwrap()).unwrap(); assert_eq!(crate::login_address::parse(&next).unwrap().unwrap().binding, binding); + assert!(withheld, "the advertised destination omits the original key"); } let links = serde_json::json!([{"rel":"data","type":"application/json","href":"https://other.example.org/collections"}]); - assert_eq!(collections_link(&address, links.as_array().unwrap()).unwrap(), "https://other.example.org/collections"); + assert_eq!(collections_link(&address, links.as_array().unwrap()).unwrap(), ("https://other.example.org/collections".into(), true)); } #[test] @@ -6301,6 +6310,18 @@ mod tests { assert_eq!(chosen.collections_url.as_deref(), Some("https://example.org/json")); } + #[test] + fn collections_links_count_only_settings_the_selected_link_omits() { + let body = br#"{"links":[{"rel":"data","type":"application/json","href":"/catalog?f=json"}]}"#; + for (from, count) in + [("https://collections.invalid/root?f=json", 0), ("https://collections.invalid/root?f=json&token=RGSYNTH_link", 1)] + { + let service = parse(from, body).unwrap(); + assert_eq!(service.collections_url.as_deref(), Some("https://collections.invalid/catalog?f=json")); + assert_eq!(service.withheld_addresses, count); + } + } + #[test] fn ogc_json_format_is_idempotent() { let url = capabilities_url("https://example.org/api?f=html&key=a%2Fb&f=json#fragment", Some("ogcapi")); diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 5c4584136..f1eed6a75 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -138,8 +138,7 @@ pub(crate) fn redacted(v: &Value) -> Value { Value::String(t) => Value::String(redacted_text(t)), Value::Array(a) => Value::Array(a.iter().map(redacted).collect()), Value::Object(o) => { - let web = |v: &Value| v.as_str().is_some_and(|t| t.trim_start().get(..4).is_some_and(|p| p.eq_ignore_ascii_case("http"))); - let beside_address = |k: &str| o.iter().any(|(sibling, v)| sibling != k && web(v)); + let beside_address = |k: &str| o.iter().any(|(sibling, v)| sibling != k && crate::project_log::address_context(sibling, v, 0)); let secret = |k: &str, v: &Value| { secret_key(k) && !matches!(v, Value::Bool(_) | Value::Null) diff --git a/crates/engine/src/cmd/notebook.rs b/crates/engine/src/cmd/notebook.rs index a60c46e7f..0f3bbfa62 100644 --- a/crates/engine/src/cmd/notebook.rs +++ b/crates/engine/src/cmd/notebook.rs @@ -181,7 +181,7 @@ pub fn commands() -> Vec { nb.cells.insert(at, Cell::new(kind, ps(p, "source").unwrap_or(""))); Ok(json!({"cell": at, "cells": nb.cells.len()})) }) - .roles(&[]) + .roles(&[crate::ParamRole { key: "source", role: crate::Role::Opaque }]) .when(need_notebook) .params(r#"{"notebook"?, "kind"?: "code"|"markdown", "source"?, "at"?: index}"#), C::new("notebook.editCell", "Edit Cell", "Notebook", |s, p| { @@ -191,7 +191,7 @@ pub fn commands() -> Vec { c.source = source.chars().take(MAX_SOURCE).collect(); Ok(json!({"cell": k})) }) - .roles(&[]) + .roles(&[crate::ParamRole { key: "source", role: crate::Role::Opaque }]) .when(need_notebook) .params(r#"{"notebook"?, "cell": index, "source": text}"#), C::new("notebook.removeCell", "Delete Cell", "Notebook", |s, p| { diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index 91ad36d15..4d7e8b617 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -103,8 +103,9 @@ struct Discovery { } impl Discovery { - fn failed(mut self, error: String) -> Result { + fn failed(mut self, s: &mut Session, error: String) -> Result { if self.service.layers.is_empty() { + notify_withheld(s, self.service.withheld_addresses); Err(error) } else { self.error = Some(error.clone()); @@ -117,7 +118,7 @@ impl Discovery { const MAX_SERVICE_DOCUMENTS: usize = 4; -fn capabilities(p: &Value) -> Result { +fn capabilities(s: &mut Session, p: &Value) -> Result { let original = url_of(p)?; let mut address = crate::render::rustgis_tiles::logins::inherit(original, &rustgis_data::ogc::capabilities_url(original, ps(p, "type"))); @@ -140,13 +141,13 @@ fn capabilities(p: &Value) -> Result { loop { crossed_origin |= crosses_origin(&parent, &address); if let Err(e) = crate::render::rustgis_tiles::address::check(&address, Some(&parent)) { - return result.failed(e); + return result.failed(s, e); } if seen.contains(&address) { - return result.failed("the service's collections links form a loop".into()); + return result.failed(s, "the service's collections links form a loop".into()); } if seen.len() >= MAX_SERVICE_DOCUMENTS { - return result.failed("the service's collections discovery exceeds four documents".into()); + return result.failed(s, "the service's collections discovery exceeds four documents".into()); } let parsed = if let Some(root) = supplied { let text = if seen.is_empty() { @@ -172,7 +173,7 @@ fn capabilities(p: &Value) -> Result { result.excluded_layers |= vetted.layers.len() != before; vetted } - Err(e) => return result.failed(e), + Err(e) => return result.failed(s, e), }; seen.insert(address.clone()); if result.service.kind.is_empty() { @@ -258,9 +259,14 @@ pub(crate) fn mark_service_source(s: &mut Session, r: &Value, url: &str) { .filter_map(Value::as_u64) .filter_map(|i| u32::try_from(i).ok()) .collect(); + let anonymous = s.web_logins.is_anonymous(); for id in ids { if let Some(l) = s.map_mut().layer_mut(id) { l.dataset.source = Some(url.to_string()); + if anonymous { + l.anonymous = true; + l.connection.clear(); + } } } } @@ -517,12 +523,12 @@ pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult { let p = owned.as_ref().unwrap_or(p); let bound = crate::service_logins::request_params(s, p)?; let p = bound.as_ref(); - let found = capabilities(p)?; + let found = capabilities(s, p)?; notify_withheld(s, found.service.withheld_addresses); let mut out = service_json(&found.service); if let Some(layers) = out.get_mut("layers").and_then(Value::as_array_mut) { for (layer, source) in layers.iter_mut().zip(found.sources) { - layer["source_url"] = json!(source.address); + layer_source(layer, source); } } if let Some(next) = found.next { @@ -531,6 +537,23 @@ pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult { Ok(out) } +/// Publishes the selected document and its already-resolved feature login provenance. +fn layer_source(layer: &mut Value, source: CapabilitySource) { + let foreign_features = layer.get("offer").is_some_and(|offer| { + offer.get("type").and_then(Value::as_str) == Some("features") + && ps(offer, "url").is_some_and(|url| crosses_origin(&source.address, url)) + }); + layer["source_url"] = json!(source.address); + if source.crossed_origin || foreign_features { + layer["anonymous"] = json!(true); + if let Some(url) = layer.get_mut("offer").and_then(|o| o.get_mut("url")) + && let Some(value) = url.as_str() + { + *url = json!(crate::render::rustgis_tiles::logins::inner(value)); + } + } +} + /// Counts destinations, not pairs; clean discovery emits no credential-forwarding notice. fn notify_withheld(s: &mut Session, count: u32) { s.notify_loss(rustgis_data::ogc::QUERY_WITHHELD_NOTICE, count); @@ -545,7 +568,7 @@ pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { let p = bound.as_ref(); let url = url_of(p)?.to_string(); let want = ps(p, "layer").map(str::trim).filter(|l| !l.is_empty()).ok_or("layer is required")?; - let found = capabilities(p)?; + let found = capabilities(s, p)?; let mut withheld = found.service.withheld_addresses; let result = add_discovered(s, p, &url, want, &found, &mut withheld); notify_withheld(s, withheld); @@ -778,29 +801,35 @@ fn add_discovered(s: &mut Session, p: &Value, url: &str, want: &str, found: &Dis } let max = crate::render::rustgis_tiles::cache::MAX_DOWNLOAD; let from = asked.as_deref().unwrap_or(items); - // An area request keeps the view until the requested box is applied below. - let keep_view = asked.is_some().then_some(false); - let mut r = match download_layer(s, &name, from, max, true).map_err(|e| slow_whole_layer(e, given))? { - Downloaded::Bytes(bytes) => { - let bytes = features_answer(s, &name, bytes, layer.bbox); - super::raster::add_named_bytes(s, &json!({"name": name}), format!("{name}.geojson"), bytes, keep_view)? + let crossed = capability_crossed || crosses_origin(source, from); + let from = if crossed { crate::render::rustgis_tiles::logins::inner(from) } else { from }; + let anonymous = crossed || s.web_logins.is_anonymous(); + crate::service_logins::anonymously(s, anonymous, |s| { + // An area request keeps the view until the requested box is applied below. + let keep_view = asked.is_some().then_some(false); + let mut r = match download_layer(s, &name, from, max, true).map_err(|e| slow_whole_layer(e, given))? { + Downloaded::Bytes(bytes) => { + let bytes = features_answer(s, &name, bytes, layer.bbox); + super::raster::add_named_bytes(s, &json!({"name": name}), format!("{name}.geojson"), bytes, keep_view)? + } + // Read in protocol buffer pages: their positions as the service's system + // places them, so there are no axes to check. + Downloaded::Read(read) => super::layer::add_read(s, &json!({"name": name}), *read, keep_view)?, + }; + mark_service_source(s, &r, from); + // Over an area, the features crossing it: the view stays, or goes to a box given + // (the whole layer's bounds would show a country to fill one state). + match (asked.is_some(), area, p.get("bbox")) { + (true, Some([w, so, e, n]), Some(Value::Array(_))) => s.zoom_to(&rustgis_geom::Extent::new(w, so, e, n)), + (true, ..) => {} + _ => zoom(s), } - // Read in protocol buffer pages: their positions as the service's system - // places them, so there are no axes to check. - Downloaded::Read(read) => super::layer::add_read(s, &json!({"name": name}), *read, keep_view)?, - }; - mark_service_source(s, &r, from); - // Over an area, the features crossing it: the view stays, or goes to a box given - // (the whole layer's bounds would show a country to fill one state). - match (asked.is_some(), area, p.get("bbox")) { - (true, Some([w, so, e, n]), Some(Value::Array(_))) => s.zoom_to(&rustgis_geom::Extent::new(w, so, e, n)), - (true, ..) => {} - _ => zoom(s), - } - if let (Some(_), Some(out)) = (&asked, r.as_object_mut()) { - out.insert("area".into(), json!(area)); - } - Ok(r) + if let (Some(_), Some(out)) = (&asked, r.as_object_mut()) { + out.insert("area".into(), json!(area)); + } + Ok(r) + }) + .unwrap_or_else(|e| std::panic::resume_unwind(e)) } } } @@ -875,7 +904,7 @@ fn coverage(s: &mut Session, p: &Value) -> CmdResult { let p = bound.as_ref(); let url = url_of(p)?.to_string(); let want = ps(p, "layer").map(str::trim).filter(|l| !l.is_empty()).ok_or("layer is required")?; - let found = capabilities(p)?; + let found = capabilities(s, p)?; let layer = found .service .layers diff --git a/crates/engine/src/cmd/service_group.rs b/crates/engine/src/cmd/service_group.rs index 300e6ddc7..f1025a6bb 100644 --- a/crates/engine/src/cmd/service_group.rs +++ b/crates/engine/src/cmd/service_group.rs @@ -81,8 +81,8 @@ fn valid_connection(s: &Session, id: &str, url: &str) -> bool { }) } -fn record_connection(s: &mut Session, layer: u32, request: &Value) { - if request.get("anonymous").and_then(Value::as_bool) == Some(true) { +fn record_connection(s: &mut Session, layer: u32, request: &Value, anonymous: bool) { + if anonymous || request.get("anonymous").and_then(Value::as_bool) == Some(true) { if let Some(layer) = s.map_mut().layer_mut(layer) { layer.anonymous = true; layer.connection.clear(); @@ -107,7 +107,8 @@ fn plan(s: &mut Session, p: &Value) -> CmdResult { fn listing(s: &mut Session, p: &Value) -> CmdResult { let q = request(s, p)?; - let mut found = capabilities(&q)?; + let mut found = capabilities(s, &q)?; + super::notify_withheld(s, found.service.withheld_addresses); if let Some(error) = found.error { return Err(error); } @@ -122,15 +123,24 @@ fn listing(s: &mut Session, p: &Value) -> CmdResult { "the service lists collections RustGIS cannot read (more than 2000, or unreadable addresses); the group was not changed".into(), ); } - let child = target(s, p)?.and_then(|(g, id)| id.and_then(|id| g.children.into_iter().find(|c| c.layer == id))); + let previous = target(s, p)?; + let child = previous.as_ref().and_then(|(g, id)| id.and_then(|id| g.children.iter().find(|c| c.layer == id).cloned())); + let anonymous: std::collections::HashSet<&str> = previous + .as_ref() + .into_iter() + .flat_map(|(g, _)| &g.children) + .filter(|c| s.map().layer(c.layer).is_some_and(|l| l.anonymous)) + .map(|c| c.collection.as_str()) + .collect(); let mut identities = std::collections::HashSet::new(); let mut offered = Vec::new(); for (layer, source) in found.service.layers.into_iter().zip(found.sources) { - let source = source.address; + let mut source = source; if !matches!(layer.offer, Offer::Features { .. }) { continue; } - if !identities.insert((source.clone(), layer.name.clone())) { + source.crossed_origin |= anonymous.contains(layer.name.as_str()); + if !identities.insert((source.address.clone(), layer.name.clone())) { return Err("the service advertises duplicate collection identifiers".into()); } offered.push((layer, source)); @@ -141,7 +151,7 @@ fn listing(s: &mut Session, p: &Value) -> CmdResult { // One child: its collection at its saved source, or else the one collection with its id (a // rotated query or a moved collections document keeps the child's identity). if let Some(c) = &child { - let exact = offered.iter().position(|(l, src)| l.name == c.collection && super::same_address(src, &c.source_url)); + let exact = offered.iter().position(|(l, src)| l.name == c.collection && super::same_address(&src.address, &c.source_url)); let pos = match exact { Some(i) => Some(i), None => { @@ -163,7 +173,7 @@ fn listing(s: &mut Session, p: &Value) -> CmdResult { let mut out = service_json(&found.service); if let Some(layers) = out.get_mut("layers").and_then(Value::as_array_mut) { for (layer, source) in layers.iter_mut().zip(sources) { - layer["source_url"] = json!(source); + super::layer_source(layer, source); } } if let Some(next) = found.next { @@ -188,7 +198,7 @@ fn download(_: &mut Session, _: &str, _: &str, _: usize) -> Result, Stri Err("the browser must supply downloaded feature responses".into()) } -fn read(s: &mut Session, p: &Value) -> Result<(Value, Vec<(ServiceChild, Dataset)>), String> { +fn read(s: &mut Session, p: &Value) -> Result<(Value, Vec<(ServiceChild, Dataset, bool)>), String> { let listing = listing(s, p)?; if listing.get("next").is_some() { return Err("complete service discovery before importing or refreshing".into()); @@ -241,6 +251,7 @@ fn read(s: &mut Session, p: &Value) -> Result<(Value, Vec<(ServiceChild, Dataset source_url: ps(layer, "source_url").unwrap_or_default().to_string(), }, dataset, + layer.get("anonymous").and_then(Value::as_bool) == Some(true), )); } Ok((listing, reads)) @@ -267,10 +278,10 @@ fn import(s: &mut Session, p: &Value) -> CmdResult { } let mut children = Vec::new(); let mut extent = rustgis_geom::Extent::EMPTY; - for (mut child, dataset) in reads { + for (mut child, dataset, anonymous) in reads { child.layer = crate::cmd::layer::add_dataset(s, dataset); if let Some(request) = listing.get("request") { - record_connection(s, child.layer, request); + record_connection(s, child.layer, request, anonymous); } if let Some(layer) = s.map().layer(child.layer) { extent = extent.union(&layer.extent()); @@ -306,7 +317,7 @@ fn refresh(s: &mut Session, p: &Value) -> CmdResult { if !s.map().service_groups.iter().any(|g| g.id == old.id) { return Err("the group was removed".into()); } - let read_count = |name: &str| reads.iter().filter(|(c, _)| c.collection == name).count(); + let read_count = |name: &str| reads.iter().filter(|(c, _, _)| c.collection == name).count(); let old_count = |name: &str| old.children.iter().filter(|c| c.collection == name).count(); // A child refresh updates that child. A group refresh keeps a child whose collection is at the // same source, or else whose collection id is unique on both sides (the source address may @@ -315,7 +326,7 @@ fn refresh(s: &mut Session, p: &Value) -> CmdResult { Some(id) => reads.iter().map(|_| Some(id)).collect(), None => reads .iter() - .map(|(child, _)| { + .map(|(child, _, _)| { old.children .iter() .find(|c| c.collection == child.collection && super::same_address(&c.source_url, &child.source_url)) @@ -333,7 +344,7 @@ fn refresh(s: &mut Session, p: &Value) -> CmdResult { let mut notes = Vec::new(); let mut singled = Vec::new(); let mut moved = 0; - for ((mut child, mut dataset), existing) in reads.into_iter().zip(matches) { + for ((mut child, mut dataset, anonymous), existing) in reads.into_iter().zip(matches) { let existing = existing.and_then(|id| s.map_mut().layer_mut(id)); if let Some(layer) = existing { if dataset.geometry_type == rustgis_data::GeometryType::None { @@ -404,12 +415,12 @@ fn refresh(s: &mut Session, p: &Value) -> CmdResult { lock.retain(|((id, _, _, _), _)| *id != child.layer); } if let Some(request) = listing.get("request") { - record_connection(s, child.layer, request); + record_connection(s, child.layer, request, anonymous); } } else { child.layer = crate::cmd::layer::add_dataset(s, dataset); if let Some(request) = listing.get("request") { - record_connection(s, child.layer, request); + record_connection(s, child.layer, request, anonymous); } if hidden && let Some(layer) = s.map_mut().layer_mut(child.layer) { layer.visible = false; diff --git a/crates/engine/src/durable.rs b/crates/engine/src/durable.rs index 39420c377..dceb7dd9c 100644 --- a/crates/engine/src/durable.rs +++ b/crates/engine/src/durable.rs @@ -21,6 +21,7 @@ pub struct Report { pub(crate) not_remembered: bool, pub(crate) lost: u32, pub(crate) not_retained: u32, + pub(crate) executable_changes: u32, } impl Report { @@ -51,6 +52,10 @@ impl Report { (self.unchecked, "step(s) contain opaque parameters that may hold credentials and were kept unchecked"), (self.not_retained, "address(es) exceeded the bridge retention limit; their keys will not survive a round trip"), (self.lost, "address(es) lost their keys when a peer sent back its project; add them again in the receiving session"), + ( + self.executable_changes, + "credential-bearing executable field(s) changed when a peer sent back its project; supply missing values before replay", + ), ] { if n > 0 { out.push(format!("{n} {text}")); @@ -81,6 +86,7 @@ impl Report { self.not_remembered |= other.not_remembered; self.lost = self.lost.saturating_add(other.lost); self.not_retained = self.not_retained.saturating_add(other.not_retained); + self.executable_changes = self.executable_changes.saturating_add(other.executable_changes); } } @@ -259,7 +265,7 @@ impl<'a> Prepare<'a> { if binding_only { kind = Kind::Runtime; } - if !binding_only { + if !binding_only || matches!(leaf, Leaf::Video(_)) { if matches!(leaf, Leaf::Video(_)) { self.report.videos = self.report.videos.saturating_add(1); } else { @@ -621,13 +627,25 @@ fn public_address_at(address: &str, depth: usize) -> Kept { let file = crate::render::rustgis_tiles::cog::split(text).0; let look = text.get(file.len()..).unwrap_or(""); let Ok(Some(b)) = s3_address::parse(file) else { return Kept::Withheld(prefix) }; - return match public_address_at(b.url, depth + 1) { - Kept::Unchanged => Kept::Unchanged, - Kept::Changed(s) => match s3_address::bind(b.binding, b.endpoint, &s) { - Ok(s) => Kept::Changed(format!("{s}{look}")), - Err(_) => Kept::Withheld(prefix), - }, - Kept::Withheld(_) => Kept::Withheld(prefix), + let endpoint = public_address_at(b.endpoint, depth + 1); + let url = public_address_at(b.url, depth + 1); + if matches!(endpoint, Kept::Withheld(_)) || matches!(url, Kept::Withheld(_)) { + return Kept::Withheld(prefix); + } + if matches!((&endpoint, &url), (Kept::Unchanged, Kept::Unchanged)) { + return Kept::Unchanged; + } + let endpoint = match &endpoint { + Kept::Changed(s) => s.as_str(), + _ => b.endpoint, + }; + let url = match &url { + Kept::Changed(s) => s.as_str(), + _ => b.url, + }; + return match s3_address::bind(b.binding, endpoint, url) { + Ok(s) => Kept::Changed(format!("{s}{look}")), + Err(_) => Kept::Withheld(prefix), }; } if matches!(prefix, "postgresql:" | "postgres:" | "pg:") { @@ -647,8 +665,38 @@ fn public_address_at(address: &str, depth: usize) -> Kept { /// Emits the report only after rollback, when the session is whole again. pub(crate) fn notify(s: &mut crate::Session, report: &Report) { for text in report.notices() { - s.notify(text, false); + s.notify(format!("Prepared output: {text}"), false); + } +} + +/// Compares credential-bearing executable leaves through the same typed output inventory. +/// Peer input never restores these credentials; a changed or removed field is reported once. +pub(crate) fn changed_executables(old: &mut crate::Project, incoming: &mut crate::Project) -> Result { + let prepared = prepared(old, None, 0)?; + let mut count = 0u32; + for edit in prepared.edits { + if edit.kind != Kind::Credential + || !matches!( + edit.at.leaf, + Leaf::Task(_) + | Leaf::ModelStep(_) + | Leaf::ModelNode(_) + | Leaf::ModelLayer(_) + | Leaf::HistoryParams + | Leaf::HistoryCommand + | Leaf::CellSource(_) + ) + { + continue; + } + let same = match (old.field_mut(&edit.at.owners, &edit.at.leaf), incoming.field_mut(&edit.at.owners, &edit.at.leaf)) { + (Some(FieldMut::Text(a)), Some(FieldMut::Text(b))) => a == b, + (Some(FieldMut::Json(a)), Some(FieldMut::Json(b))) => a == b, + _ => false, + }; + count = count.saturating_add(u32::from(!same)); } + Ok(count) } impl Durable for Vec { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 9159a673d..299b1c4d4 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -16,7 +16,7 @@ pub mod prefs; pub mod project; mod roles; pub use peer::Peer; -pub use roles::{Document, ParamRole, Role}; +pub(crate) use roles::{Document, ParamRole, Role}; mod parameters; pub mod project_log; pub use parameters::{params_envelope, script_line}; @@ -66,8 +66,8 @@ pub struct CommandSpec { /// `None` = enabled; `Some(reason)` = disabled. pub enabled: fn(&Session) -> Option<&'static str>, pub run: fn(&mut Session, &Value) -> CmdResult, - /// Durable-output roles. None keeps unreviewed extension parameters opaque. - pub roles: Option<&'static [ParamRole]>, + /// Reviewed builtin durable-output roles. Extension parameters remain opaque. + pub(crate) roles: Option<&'static [ParamRole]>, } fn always(_: &Session) -> Option<&'static str> { @@ -89,8 +89,8 @@ impl CommandSpec { pub const fn new(id: &'static str, label: &'static str, location: &'static str, run: fn(&mut Session, &Value) -> CmdResult) -> Self { CommandSpec { id, label, location, shortcut: "", params: "{}", mutates: true, enabled: always, run, roles: None } } - /// Declares durable-output roles; an empty list uses reviewed common parameter rules. - pub const fn roles(mut self, roles: &'static [ParamRole]) -> Self { + /// Declares builtin durable-output roles; an empty list uses reviewed common parameter rules. + pub(crate) const fn roles(mut self, roles: &'static [ParamRole]) -> Self { self.roles = Some(roles); self } diff --git a/crates/engine/src/peer.rs b/crates/engine/src/peer.rs index 099c2fc6d..58b4e7a9c 100644 --- a/crates/engine/src/peer.rs +++ b/crates/engine/src/peer.rs @@ -6,23 +6,52 @@ use crate::{ durable::{self, Report}, }; use durable::{At, Durable, FieldMut, Kept, Leaf, Owner}; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; #[derive(Clone, Debug, PartialEq, Eq, Hash)] -enum SlotKey { - Map { index: usize, name: String, at: rustgis_map::SlotAt }, - Area { index: usize, at: rustgis_map::SlotAt }, +enum Position<'a> { + Layer { id: u32, role: rustgis_map::LayerRole }, + Tile { id: Cow<'a, str>, role: rustgis_map::TileRole }, + Group { id: u32, child: Option }, + Terrain, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +enum SlotKey<'a> { + Map { index: usize, name: Cow<'a, str>, at: Position<'a> }, + Area { index: usize, at: Position<'a> }, Table(u32), Video { presentation: usize, page: usize }, } +impl SlotKey<'_> { + /// Only retained, budgeted identities own copies; raw censuses borrow hostile names. + fn owned(&self) -> SlotKey<'static> { + let position = |at: &Position<'_>| match at { + Position::Layer { id, role } => Position::Layer { id: *id, role: *role }, + Position::Tile { id, role } => Position::Tile { id: Cow::Owned(id.to_string()), role: *role }, + Position::Group { id, child } => Position::Group { id: *id, child: *child }, + Position::Terrain => Position::Terrain, + }; + match self { + Self::Map { index, name, at } => SlotKey::Map { index: *index, name: Cow::Owned(name.to_string()), at: position(at) }, + Self::Area { index, at } => SlotKey::Area { index: *index, at: position(at) }, + Self::Table(id) => SlotKey::Table(*id), + Self::Video { presentation, page } => SlotKey::Video { presentation: *presentation, page: *page }, + } + } +} + struct Slot<'a> { - key: SlotKey, + key: SlotKey<'a>, at: At, value: &'a str, + layer: Option<&'a rustgis_map::Layer>, } struct Entry { - key: SlotKey, + key: SlotKey<'static>, + choice: Option, public: String, full: String, ambiguous: bool, @@ -40,29 +69,42 @@ fn slots(p: &Project) -> Vec> { let mut out = Vec::new(); for (i, m) in p.maps.iter().enumerate() { for slot in m.slots().into_iter().filter(|s| s.address) { - if let Some(at) = m.slot_identity(&slot.at) { + if let Some(at) = position(m, &slot.at) { out.push(Slot { - key: SlotKey::Map { index: i, name: m.name.clone(), at }, - at: At { owners: vec![Owner::Map(i)], leaf: Leaf::Map(slot.at) }, + key: SlotKey::Map { index: i, name: Cow::Borrowed(&m.name), at }, + at: At { owners: vec![Owner::Map(i)], leaf: Leaf::Map(slot.at.clone()) }, value: slot.value, + layer: match slot.at { + rustgis_map::SlotPos::Layer { layer, .. } => m.layers.get(layer), + _ => None, + }, }); } } } for (i, area) in p.offline_areas.iter().enumerate() { for slot in area.data.slots().into_iter().filter(|s| s.address) { - if let Some(at) = area.data.slot_identity(&slot.at) { + if let Some(at) = position(&area.data, &slot.at) { out.push(Slot { key: SlotKey::Area { index: i, at }, - at: At { owners: vec![Owner::Area(i)], leaf: Leaf::Map(slot.at) }, + at: At { owners: vec![Owner::Area(i)], leaf: Leaf::Map(slot.at.clone()) }, value: slot.value, + layer: match slot.at { + rustgis_map::SlotPos::Layer { layer, .. } => area.data.layers.get(layer), + _ => None, + }, }); } } } for (i, table) in p.tables.iter().enumerate() { if let Some(value) = &table.source { - out.push(Slot { key: SlotKey::Table(table.id), at: At { owners: vec![Owner::Table(i)], leaf: Leaf::Table }, value }); + out.push(Slot { + key: SlotKey::Table(table.id), + at: At { owners: vec![Owner::Table(i)], leaf: Leaf::Table }, + value, + layer: None, + }); } } for (i, pres) in p.presentations.iter().enumerate() { @@ -72,6 +114,7 @@ fn slots(p: &Project) -> Vec> { key: SlotKey::Video { presentation: i, page }, at: At { owners: vec![Owner::Presentation(i)], leaf: Leaf::Video(page) }, value: source, + layer: None, }); } } @@ -79,6 +122,88 @@ fn slots(p: &Project) -> Vec> { out } +/// Resolves raw owner identity without copying untrusted map or tile names. +fn position<'a>(map: &'a rustgis_map::Map, at: &rustgis_map::SlotPos) -> Option> { + use rustgis_map::SlotPos; + Some(match at { + SlotPos::Layer { layer, role } => Position::Layer { id: map.layers.get(*layer)?.id, role: *role }, + SlotPos::Tile { tile, role } => Position::Tile { id: Cow::Borrowed(&map.tiles.get(*tile)?.id), role: *role }, + SlotPos::Group { group, child } => { + let group = map.service_groups.get(*group)?; + Position::Group { + id: group.id, + child: match child { + Some(i) => Some(group.children.get(*i)?.layer), + None => None, + }, + } + } + SlotPos::Terrain => Position::Terrain, + }) +} + +/// Read selection, not drawing style or cached rows. The last exchange, live source and raw +/// incoming source must agree before normalization or a full address can be retained. +fn source_choice(layer: Option<&rustgis_map::Layer>) -> Result, ()> { + let Some(layer) = layer.filter(|l| l.source.is_some() || l.raster.is_some() || l.postgis.is_some()) else { return Ok(None) }; + if layer.output_slots().iter().any(|(_, value, _, _)| value.len() > MAX_BYTES) + || layer.source.as_ref().and_then(|s| s.embedded.as_ref()).is_some_and(|b| b.len() > MAX_BYTES) + || layer.raster.as_ref().and_then(|r| r.embedded.as_ref()).is_some_and(|b| b.len() > MAX_BYTES) + { + return Err(()); + } + let public = |s: &str| match durable::public_address(s) { + Kept::Unchanged | Kept::Withheld(_) => s.to_string(), + Kept::Changed(s) => s, + }; + let source = layer.source.as_ref().map(|s| { + ( + s.path.as_deref().map(public), + &s.file_name, + &s.table, + &s.shapefile_crs, + s.embedded.as_deref(), + s.gdal.as_ref().map(|g| (public(&g.path), &g.layer, g.family, &g.crs)), + s.duckdb.as_ref().map(|d| (&d.sql, d.database.as_deref().map(public), &d.table, &d.crs, d.family)), + s.postgis.as_ref().map(|d| (public(&d.database), &d.table, d.family, &d.crs)), + s.mysql.as_ref().map(|d| (public(&d.database), &d.table, d.family, &d.crs)), + s.sqlserver.as_ref().map(|d| (public(&d.database), &d.table, d.family, &d.crs)), + ) + }); + let raster = layer.raster.as_ref().map(|r| { + ( + r.path.as_deref().map(public), + &r.file_name, + r.embedded.as_deref(), + &r.subdataset, + &r.crs, + r.placement, + r.time_zarr.as_ref().map(|z| (public(&z.store), &z.variable, &z.index, &z.dim, z.offset, &z.dates, &z.crs)), + ) + }); + let edit = layer.postgis.as_ref().map(|p| { + (public(&p.origin.database), &p.origin.table, p.origin.family, &p.origin.crs, &p.key, p.srid, p.writable, &p.condition, p.limit) + }); + struct Bounded(Vec); + impl std::io::Write for Bounded { + /// Stops at the retention budget before allocating another chunk. + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.0.len().saturating_add(bytes.len()) > MAX_BYTES { + return Err(std::io::Error::other("peer source identity exceeds retention limit")); + } + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + /// The bounded buffer has no pending I/O. + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut out = Bounded(Vec::new()); + serde_json::to_writer(&mut out, &(source, raster, edit)).map_err(|_| ())?; + String::from_utf8(out.0).map(Some).map_err(|_| ()) +} + /// A wrapper's binding belongs to S3 connection resolution, never exchange retention. fn wrapped(value: &str) -> bool { let value = crate::sources::template_archive(value).map_or(value, |(_, v)| v); @@ -86,16 +211,16 @@ fn wrapped(value: &str) -> bool { } /// Charges both forms and the complete identity payload, bounded before copying either form. -fn key_bytes(key: &SlotKey) -> usize { +fn key_bytes(key: &SlotKey<'_>) -> usize { let extra = match key { SlotKey::Map { name, at, .. } => name.len().saturating_add(match at { - rustgis_map::SlotAt::Tile { id, .. } => id.len(), + Position::Tile { id, .. } => id.len(), _ => 0, }), - SlotKey::Area { at: rustgis_map::SlotAt::Tile { id, .. }, .. } => id.len(), + SlotKey::Area { at: Position::Tile { id, .. }, .. } => id.len(), _ => 0, }; - std::mem::size_of::().saturating_add(extra) + std::mem::size_of::>().saturating_add(extra) } impl Exchange { @@ -120,7 +245,16 @@ impl Exchange { if let Some((first, ambiguous)) = forms.get_mut(&public) { *ambiguous |= entries.get(*first).is_none_or(|e| e.full != slot.value); } - let bytes = slot.value.len().saturating_add(public.len()).saturating_add(key_bytes(&slot.key)); + let Ok(choice) = source_choice(slot.layer) else { + omitted = omitted.saturating_add(1); + continue; + }; + let bytes = slot + .value + .len() + .saturating_add(public.len()) + .saturating_add(key_bytes(&slot.key)) + .saturating_add(choice.as_ref().map_or(0, String::len)); if !recording || entries.len() >= MAX_ENTRIES || held.saturating_add(bytes) > MAX_BYTES { recording = false; omitted = omitted.saturating_add(1); @@ -128,7 +262,7 @@ impl Exchange { } held += bytes; forms.entry(public.clone()).or_insert((entries.len(), false)); - entries.push(Entry { key: slot.key, public, full: slot.value.into(), ambiguous: false }); + entries.push(Entry { key: slot.key.owned(), choice, public, full: slot.value.into(), ambiguous: false }); } for entry in &mut entries { entry.ambiguous = forms.get(&entry.public).is_none_or(|(_, ambiguous)| *ambiguous); @@ -137,24 +271,28 @@ impl Exchange { } /// The pre-normalization census must contain exactly one byte-equal public slot. - fn incoming(&self, p: &Project) -> HashSet { + fn incoming(&self, p: &Project) -> HashSet> { let census = census(p); self.entries .iter() - .filter(|e| census.get(&e.key).is_some_and(|(n, value)| *n == 1 && *value == e.public)) + .filter(|e| { + census.get(&e.key).is_some_and(|(n, value, layer)| { + *n == 1 && *value == e.public && source_choice(*layer).ok().as_ref() == Some(&e.choice) + }) + }) .map(|e| e.key.clone()) .collect() } /// Checks last exchange, both incoming censuses, unique account and the exact current live /// value in one synchronous replacement. Public URL or incoming id alone grants nothing. - fn retain(&self, live: &Project, incoming: &mut Project, raw: &HashSet) { + fn retain(&self, live: &Project, incoming: &mut Project, raw: &HashSet>) { let current = census(live); let normalized = census(incoming); - let mut records: HashMap<&SlotKey, (usize, &Entry)> = HashMap::new(); + let mut records: HashMap, (usize, &Entry)> = HashMap::new(); let mut accounts: HashMap<&str, Option<&str>> = HashMap::new(); for e in &self.entries { - records.entry(&e.key).or_insert((0, e)).0 += 1; + records.entry(e.key.clone()).or_insert((0, e)).0 += 1; let account = accounts.entry(&e.public).or_insert(Some(&e.full)); if *account != Some(e.full.as_str()) { *account = None; @@ -164,11 +302,13 @@ impl Exchange { .into_iter() .filter_map(|slot| { let (count, e) = records.get(&slot.key)?; - let unique = |census: &HashMap, expected: &str| { - census.get(&slot.key).is_some_and(|(n, value)| *n == 1 && *value == expected) + let unique = |census: &Census<'_>, expected: &str| { + census.get(&slot.key).is_some_and(|(n, value, layer)| { + *n == 1 && *value == expected && source_choice(*layer).ok().as_ref() == Some(&e.choice) + }) }; (!e.ambiguous - && raw.contains(&slot.key) + && raw.contains(&e.key) && *count == 1 && accounts.get(e.public.as_str()) == Some(&Some(e.full.as_str())) && unique(¤t, &e.full) @@ -221,10 +361,13 @@ fn reopen(p: &mut Project, at: &At) { } /// Values borrow the project, but identities include all duplicates; no normalization runs here. -fn census(p: &Project) -> HashMap { +type Census<'a> = HashMap, (usize, &'a str, Option<&'a rustgis_map::Layer>)>; + +/// Counts raw duplicates while borrowing all untrusted identity text. +fn census(p: &Project) -> Census<'_> { let mut out = HashMap::new(); for slot in slots(p) { - let entry = out.entry(slot.key).or_insert((0usize, slot.value)); + let entry = out.entry(slot.key).or_insert((0usize, slot.value, slot.layer)); entry.0 = entry.0.saturating_add(1); } out @@ -240,6 +383,7 @@ impl Session { /// Writes a credential-free project for a peer, without evicting explicit-save restoration. /// Pending downloads require a structural draft; settled projects use the live journal. pub fn peer_text(&mut self, peer: Peer) -> Result<(String, Report), String> { + let runtime = self.project.bound_address_losses(true); let (text, mut report) = if self.project.ensure_imports_complete().is_err() { let mut draft = self.project.clone(); draft.omit_pending_imports(); @@ -247,6 +391,7 @@ impl Session { } else { durable::output(&mut self.project, None, text) }?; + report.addresses = report.addresses.saturating_add(u32::try_from(runtime).unwrap_or(u32::MAX)); self.exchange = if peer == Peer::Bridge { let (record, omitted) = Exchange::record(&self.project, self.opened); report.not_retained = omitted; @@ -275,12 +420,13 @@ impl Session { .into_iter() .filter(|slot| { !matches!(durable::public_address(slot.value), Kept::Unchanged) - && !incoming.get(&slot.key).is_some_and(|(count, value)| *count == 1 && *value == slot.value) + && !incoming.get(&slot.key).is_some_and(|(count, value, _)| *count == 1 && *value == slot.value) }) .count() .try_into() .unwrap_or(u32::MAX); drop(incoming); + report.executable_changes = durable::changed_executables(&mut self.project, &mut p)?; crate::cmd::project::reset_to(self, p); self.path = None; crate::cmd::connections::resolve_project(self); @@ -303,3 +449,39 @@ fn text(p: &Project) -> Result { } Ok(text) } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + /// Table/family/CRS are part of a retained reader, even before normalization opens it. + #[test] + fn bridge_census_binds_the_full_source_choice_before_normalization() { + let mut p = Project::default(); + let id = p.map_mut().add_dataset(rustgis_data::Dataset::new("Database", rustgis_data::GeometryType::None, vec![])); + let mut source = rustgis_map::source::SourceLayer::default(); + source.postgis = Some(rustgis_map::source::PostgisOrigin { + database: "postgresql://reader:RGSYNTH_peer_choice@db.invalid/gis".into(), + table: "approved".into(), + family: None, + crs: None, + }); + p.map_mut().layer_mut(id).unwrap().source = Some(source); + let mut s = Session::new(p); + let (public, _) = s.peer_text(Peer::Bridge).unwrap(); + let exchange = s.exchange.as_ref().unwrap(); + for change in ["style", "table", "family", "crs"] { + let mut p = Project::from_peer_raw(&public).unwrap(); + let layer = &mut p.maps[0].layers[0]; + let pg = layer.source.as_mut().unwrap().postgis.as_mut().unwrap(); + match change { + "table" => pg.table = "different".into(), + "family" => pg.family = Some(rustgis_data::source::Family::Points), + "crs" => pg.crs = Some("EPSG:4326".into()), + _ => layer.visible = false, + } + assert_eq!(exchange.incoming(&p).len(), usize::from(change == "style"), "{change}"); + } + } +} diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 06e9d7b43..9243b1c47 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -653,30 +653,35 @@ impl Project { addresses } - /// How many layers and tile layers read through a login binding but record no connection: - /// nothing would tell the opened project which login they need. One whose address also holds - /// a token isn't counted here: the durable address projection counts it already. - fn unrecorded_bound(&mut self) -> usize { + /// Runtime bindings that this output cannot restore: peer receivers never share a login; + /// explicit files can resolve layers and tile owners with a recorded saved connection. + pub(crate) fn bound_address_losses(&self, include_recorded: bool) -> usize { let only_bound = |a: &str| { without_login_binding(a) .is_some_and(|public| matches!(crate::durable::public_address(&public), crate::durable::Kept::Unchanged)) }; - let mut n = 0; - for map in self.maps.iter_mut().chain(self.offline_areas.iter_mut().map(|area| &mut area.data)) { - let tiles: std::collections::BTreeSet<_> = map - .slots() - .into_iter() - .filter_map(|slot| { - let rustgis_map::SlotPos::Tile { tile, .. } = slot.at else { return None }; - map.tiles.get(tile).filter(|t| t.connection.is_empty() && only_bound(slot.value)).map(|_| tile) - }) - .collect(); - n += tiles.len(); - for layer in map.layers.iter_mut().filter(|l| l.connection.is_empty()) { - n += usize::from(layer.addresses_mut().any(|a| only_bound(a))); + let mut n = 0usize; + for map in self.maps.iter().chain(self.offline_areas.iter().map(|area| &area.data)) { + let mut layers = std::collections::BTreeSet::new(); + let mut tiles = std::collections::BTreeSet::new(); + for slot in map.slots().into_iter().filter(|slot| slot.address && only_bound(slot.value)) { + match slot.at { + rustgis_map::SlotPos::Layer { layer, .. } => { + if map.layers.get(layer).is_some_and(|l| include_recorded || l.connection.is_empty()) { + layers.insert(layer); + } + } + rustgis_map::SlotPos::Tile { tile, .. } => { + if map.tiles.get(tile).is_some_and(|t| include_recorded || t.connection.is_empty()) { + tiles.insert(tile); + } + } + rustgis_map::SlotPos::Group { .. } | rustgis_map::SlotPos::Terrain => n = n.saturating_add(1), + } } + n = n.saturating_add(layers.len()).saturating_add(tiles.len()); } - n + n.saturating_add(self.tables.iter().filter(|table| table.source.as_deref().is_some_and(only_bound)).count()) } /// The standalone table of an id. @@ -948,7 +953,7 @@ impl Project { /// fields are restored on every return and native unwind; only explicit saves are remembered. pub fn saved_text(&mut self, dir: Option<&Path>) -> Result<(String, crate::durable::Report), String> { self.ensure_imports_complete()?; - let unrecorded = self.unrecorded_bound(); + let unrecorded = self.bound_address_losses(false); let (text, mut report) = crate::durable::output_remembered(self, dir.and_then(Path::to_str), project_text)?; report.addresses = report.addresses.saturating_add(u32::try_from(unrecorded).unwrap_or(u32::MAX)); Ok((text, report)) @@ -970,7 +975,9 @@ impl Project { /// Text too large to open again is refused. Returns the text and its counted output notices. pub fn recovery_text_with_report(mut self) -> Result<(String, crate::durable::Report), String> { self.omit_pending_imports(); - let (text, report) = crate::durable::output_owned(self, project_text)?; + let unrecorded = self.bound_address_losses(false); + let (text, mut report) = crate::durable::output_owned(self, project_text)?; + report.addresses = report.addresses.saturating_add(u32::try_from(unrecorded).unwrap_or(u32::MAX)); if text.len() as u64 > MAX_BYTES { return Err("the project is too large to keep a recovery copy".into()); } diff --git a/crates/engine/src/project_log.rs b/crates/engine/src/project_log.rs index 431d5dd8f..57b73dde5 100644 --- a/crates/engine/src/project_log.rs +++ b/crates/engine/src/project_log.rs @@ -135,6 +135,25 @@ fn redacted(v: &Value) -> Value { crate::cmd::history::redacted(&embedded_redacted(v, 0)) } +/// A bounded log-only context check for a short key beside nested connection settings. +/// User data/documents do not make an ordinary join or lookup key a credential. +pub(crate) fn address_context(key: &str, v: &Value, depth: usize) -> bool { + if depth >= 8 || matches!(key, "data" | "text" | "values" | "attributes" | "properties" | "features" | "geometry") { + return false; + } + match v { + Value::String(s) => { + let s = s.trim_start(); + ["http://", "https://", "s3://", "rustgis-login:", "rustgis-s3:", "postgres:", "postgresql:", "mysql:", "mssql:"] + .iter() + .any(|prefix| s.get(..prefix.len()).is_some_and(|head| head.eq_ignore_ascii_case(prefix))) + } + Value::Array(values) => values.iter().take(LOG_LIST_ITEMS).any(|v| address_context("", v, depth + 1)), + Value::Object(values) => values.iter().any(|(key, v)| address_context(key, v, depth + 1)), + _ => false, + } +} + /// Scrubs bounded nested log parameters and JSON embedded in text before the log retains them. fn embedded_redacted(v: &Value, depth: usize) -> Value { match v { diff --git a/crates/engine/src/roles.rs b/crates/engine/src/roles.rs index 7574e4fee..3d7c069fa 100644 --- a/crates/engine/src/roles.rs +++ b/crates/engine/src/roles.rs @@ -12,18 +12,17 @@ use crate::durable::{self, Durable, Kept, Report}; /// A replayable parameter's durable-output role. Permission policy does not consume roles. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ParamRole { +pub(crate) struct ParamRole { pub key: &'static str, pub role: Role, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Role { - Address, - Path, +pub(crate) enum Role { Secret, Dataset, Configuration, Sql, + Opaque, ToolParams, Script, TaskSteps, @@ -32,7 +31,7 @@ pub enum Role { Document(Document), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Document { +pub(crate) enum Document { Project, Map, Task, @@ -77,7 +76,7 @@ const ADDRESS: &[&str] = &["url", "urls", "href", "source", "template", "tiles", "store", "endpoint", "address", "uri", "connection", "database"]; const CONFIG: &[&str] = &["headers", "options", "auth", "connection", "source", "sources"]; -/// Durable output reads its own field from the immutable command registry. An explicit empty +/// Durable output reads its own field from reviewed builtin registrations. An explicit empty /// declaration retains the reviewed common scalar/address/configuration rules below. fn metadata(id: &str) -> Option<&'static [ParamRole]> { static ROLES: OnceLock> = OnceLock::new(); @@ -149,6 +148,9 @@ fn object(id: &str, value: &Value, depth: usize, config_depth: usize, registered } else { envelope(role, v, obj, depth + 1, report) } + } else if role == Some(Role::Opaque) { + unchecked |= suspicious(v); + None } else if (role == Some(Role::Secret) || rustgis_data::address::secret_key(key)) && (k != "key" || crate::sources::looks_like_a_key(v.as_str().unwrap_or("")) @@ -175,7 +177,7 @@ fn object(id: &str, value: &Value, depth: usize, config_depth: usize, registered let clean = crate::project_log::sql_secrets_removed(s); (clean != s).then_some(Value::String(clean)) }) - } else if matches!(role, Some(Role::Address | Role::Path)) || ADDRESS.contains(&k.as_str()) || address_like(v) { + } else if ADDRESS.contains(&k.as_str()) || address_like(v) { if v.is_object() && registered && (role == Some(Role::Configuration) || CONFIG.contains(&k.as_str())) { object("", v, depth, config_depth + 1, true, report) } else { diff --git a/crates/engine/src/tests/credential_output.rs b/crates/engine/src/tests/credential_output.rs index d9cc5be2a..7163d3c22 100644 --- a/crates/engine/src/tests/credential_output.rs +++ b/crates/engine/src/tests/credential_output.rs @@ -123,6 +123,12 @@ fn durable_empty_query_pairs_preserve_clean_urls_and_report_real_loss() { ("layer=1&", "layer=1&", 0), ("layer=1 \t", "layer=1 \t", 0), ("layer=1&&token=RGSYNTH_empty_query&", "layer=1", 1), + ("layer=1&appid=RGSYNTH_empty_query", "layer=1", 1), + ("layer=1&%61ppid=RGSYNTH_empty_query", "layer=1", 1), + ("layer=1&subscription-key=RGSYNTH_empty_query", "layer=1", 1), + ("layer=1&client_key=RGSYNTH_empty_query", "layer=1", 1), + ("layer=1&sas-key=RGSYNTH_empty_query", "layer=1", 1), + ("client_id=public&application_id=public&map=1", "client_id=public&application_id=public&map=1", 0), ] { let url = format!("https://h.invalid/tiles/{{z}}/{{x}}/{{y}}.png?{query}"); let mut project = Project::default(); @@ -559,6 +565,7 @@ fn durable_database_origins_clean_each_owned_field_without_opening_a_database() ("postgis", "postgresql", "/source/postgis/database"), ("mysql", "mysql", "/source/mysql/database"), ("sqlserver", "mssql", "/source/sqlserver/database"), + ("duckdb", "postgresql", "/source/duckdb/database"), ] { for private in [false, true] { let public = format!("{scheme}://reader@db-origin.invalid/gis"); @@ -566,7 +573,10 @@ fn durable_database_origins_clean_each_owned_field_without_opening_a_database() if private { format!("{scheme}://reader:RGSYNTH_db_origin_20261106@db-origin.invalid/gis") } else { public.clone() }; let dataset = rustgis_data::Dataset::new("Origin", rustgis_data::GeometryType::None, Vec::new()); let mut layer = serde_json::to_value(rustgis_map::Layer::new(1, dataset)).unwrap(); - let origin = json!({"database":address, "table":"rows"}); + let mut origin = json!({"database":address, "table":"rows"}); + if role == "duckdb" { + origin["family"] = json!("Points"); + } if role == "edit" { layer["postgis"] = json!({"origin":origin, "key":"id", "srid":4326, "writable":false}); } else { @@ -606,3 +616,139 @@ fn durable_database_origins_clean_each_owned_field_without_opening_a_database() } } } + +/// Runtime-only login omissions and executable replacement losses use different notice kinds. +#[test] +fn peer_output_counts_runtime_logins_and_changed_executable_fields() { + let public = "https://peer-runtime.invalid/data"; + let bound = rustgis_data::login_address::bind("0123456789abcdef", "https://peer-runtime.invalid", public).unwrap(); + let mut s = Session::new(Project::default()); + let mut dataset = rustgis_data::Dataset::new("Bound", rustgis_data::GeometryType::None, Vec::new()); + dataset.source = Some(bound.clone()); + let id = s.map_mut().add_dataset(dataset); + s.map_mut().layer_mut(id).unwrap().connection = "saved-connection".into(); + s.project.tasks.push(Task { + name: "Login".into(), + description: String::new(), + steps: vec![TaskStep { + title: String::new(), + instructions: String::new(), + command: "connection.login".into(), + params: json!({"apiKey":SECRET}), + }], + }); + let (text, report) = s.peer_text(Peer::Embedder).unwrap(); + assert_eq!(report.layer_addresses(), 1, "a foreign receiver cannot use the recorded connection's live binding"); + assert!(!text.contains("rustgis-login:") && !text.contains(SECRET)); + assert_eq!(s.map().layer(id).unwrap().dataset.source.as_deref(), Some(bound.as_str())); + let report = s.open_peer_text(Peer::Embedder, &text).unwrap(); + assert_eq!(report.executable_changes, 1); + assert!(report.notices().iter().any(|n| n.contains("executable field(s) changed"))); + assert_eq!(s.project.tasks[0].steps[0].params["apiKey"], ""); +} + +/// A valid S3 wrapper's endpoint is also an address, not an opaque place to store a password. +#[test] +fn durable_s3_endpoints_clean_recognized_credentials_and_keep_provenance() { + for (endpoint, expected, count) in [ + ("https://reader:RGSYNTH_endpoint@h.invalid", "https://h.invalid", 1), + ("https://h.invalid?token=RGSYNTH_endpoint", "https://h.invalid", 1), + ("https://h.invalid", "https://h.invalid", 0), + ("", "", 0), + ] { + let address = rustgis_data::s3_address::bind("0123456789abcdef", endpoint, "s3://bucket/data").unwrap(); + let mut p = Project::default(); + let mut table = rustgis_data::Dataset::new("S3", rustgis_data::GeometryType::None, Vec::new()); + table.source = Some(address.clone()); + p.add_table(table); + let (text, report) = p.saved_text(None).unwrap(); + assert!(!text.contains("RGSYNTH_endpoint")); + assert_eq!(report.layer_addresses(), count); + let value: Value = serde_json::from_str(&text).unwrap(); + let parsed = rustgis_data::s3_address::parse(value["tables"][0]["source"].as_str().unwrap()).unwrap().unwrap(); + assert_eq!((parsed.binding, parsed.endpoint, parsed.url), ("0123456789abcdef", expected, "s3://bucket/data")); + assert_eq!(p.tables[0].source.as_deref(), Some(address.as_str())); + } +} + +#[test] +fn nested_notebook_source_stays_opaque_but_is_reported_unchecked() { + for command in ["notebook.addCell", "notebook.editCell"] { + for source in ["layer.addUrl {\"url\":\"https://h.invalid/?token=RGSYNTH_nested\"}", "print('RGSYNTH_nested token')", "clean prose"] + { + let mut p = Project::default(); + p.tasks.push(Task { + name: "Nested".into(), + description: String::new(), + steps: vec![TaskStep { + title: String::new(), + instructions: String::new(), + command: command.into(), + params: json!({"source":source}), + }], + }); + let (text, report) = p.saved_text(None).unwrap(); + let written: Value = serde_json::from_str(&text).unwrap(); + assert_eq!(written["tasks"][0]["steps"][0]["params"]["source"], source); + assert_eq!(report.unchecked, u32::from(source != "clean prose")); + assert_eq!(report.steps, 0); + } + } + assert_eq!(crate::log_params(&json!({"key":"abc", "source":{"url":"https://h.invalid/a"}})).into_log_value()["key"], ""); + assert_eq!(crate::log_params(&json!({"key":"abc", "data":{"url":"https://h.invalid/a"}})).into_log_value()["key"], "abc"); +} + +/// A retained address must actually reopen its protected reader, not merely survive as text. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn bridge_round_trip_reopens_its_protected_reader() { + if super::isolate_account_test("tests::credential_output::bridge_round_trip_reopens_its_protected_reader") { + return; + } + use rustgis_data::source::{fgb, remote}; + let (dataset, _) = rustgis_data::io::read_geojson(r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1,2]},"properties":{"value":7}}]}"#, "Protected").unwrap(); + let bytes = fgb::write(&dataset).unwrap(); + let original = fgb::FgbSource::open_bytes(bytes.clone(), "Protected").unwrap(); + let full = "https://protected-reader.invalid/data.fgb?token=RGSYNTH_reader"; + // Session construction installs the native fetcher; replace it only after that setup. + let mut s = Session::new(Project::default()); + remote::set_fetcher(move |url, want, done| { + if url != full { + done(Err(remote::Failed::Other("protected reader needs its token".into()))); + } else if matches!(want, remote::Want::Whole) { + done(Ok(bytes.clone())); + } else { + done(Err(remote::Failed::NoRanges)); + } + }); + let id = s.map_mut().add_dataset(dataset); + s.map_mut().layer_mut(id).unwrap().source = Some(rustgis_map::source::SourceLayer::from_path(full, None, Box::new(original))); + let (text, _) = s.peer_text(Peer::Bridge).unwrap(); + assert!(!text.contains("RGSYNTH_reader")); + let report = s.open_peer_text(Peer::Bridge, &text).unwrap(); + assert_eq!(report.lost, 0, "the unchanged streamed source retains its address"); + let source = s.map_mut().layer_mut(id).unwrap().source.as_mut().unwrap(); + assert_eq!(source.path.as_deref(), Some(full), "the exact source keeps its protected address"); + source.wait_open(std::time::Duration::from_secs(5)); + assert!(source.is_open(), "retained reader failed to open: {:?}", source.error); + let (read, _) = source.read_all().unwrap(); + assert_eq!(read.features.len(), 1); + assert_eq!(read.features[0].attrs[0], rustgis_data::Value::Int(7)); +} + +/// Discovery must announce omitted settings even when the following supplied document is invalid. +#[test] +fn failed_collections_discovery_retains_its_counted_query_notice() { + for (query, count) in [("", 0), ("?token=RGSYNTH_discovery", 1)] { + let mut s = Session::new(Project::default()); + let p = json!({"url":format!("https://discovery.invalid/root{query}"), "type":"ogcapi", + "text":json!({"links":[{"rel":"data", "type":"application/json", "href":"/catalog"}]}).to_string(), + "documents":[{"url":"https://discovery.invalid/catalog", "text":"invalid capabilities"}]}); + assert!(s.run("service.layers", &p).is_err()); + let messages: Vec<_> = s.notices.iter().filter(|n| n.text.contains("address(es) the service listed")).collect(); + assert_eq!(messages.len(), count); + if count > 0 { + assert!(messages[0].text.starts_with("1 address(es)")); + } + } +} diff --git a/crates/engine/src/tests/service_logins.rs b/crates/engine/src/tests/service_logins.rs index 404785318..0a5b63e4b 100644 --- a/crates/engine/src/tests/service_logins.rs +++ b/crates/engine/src/tests/service_logins.rs @@ -1602,3 +1602,57 @@ fn capabilities_styles_keep_foreign_lineage() { } assert!(results.iter().all(|(_, _, kept)| *kept), "style discovery forgot its foreign hop: {results:?}"); } + +/// Features reached through foreign capabilities keep the same opt-out as tile discovery. +#[test] +fn feature_group_plans_and_imports_keep_foreign_capabilities_anonymous() { + #[cfg(not(target_arch = "wasm32"))] + if super::isolate_account_test("tests::service_logins::feature_group_plans_and_imports_keep_foreign_capabilities_anonymous") { + return; + } + for (document_crossed, items_crossed, authenticated) in + [(false, false, false), (false, false, true), (true, false, true), (false, true, true)] + { + let crossed = document_crossed || items_crossed; + let mut s = session(None); + let a = "https://feature-lineage-a.invalid"; + let b = "https://feature-lineage-b.invalid"; + let bridge = if document_crossed { format!("{b}/bridge") } else { format!("{a}/bridge") }; + let items = if items_crossed { format!("{b}/items") } else { format!("{a}/items") }; + let login = new(&mut s, json!({"kind":"ogcapi", "address":a, "auth":if authenticated {"header"} else {"none"}, "keyName":"X-Key"})); + if authenticated { + s.run("connection.login", &json!({"id":login, "apiKey":"RGSYNTH_feature_lineage"})).unwrap(); + } + let other = new(&mut s, json!({"kind":"ogcapi", "address":b, "auth":"header", "keyName":"X-Key"})); + s.run("connection.login", &json!({"id":other, "apiKey":"RGSYNTH_feature_destination"})).unwrap(); + let link = |url: &str| json!({"links":[{"rel":"data", "type":"application/json", "href":url}]}).to_string(); + let mut params = json!({"url":format!("{a}/root"), "type":"ogcapi", "connection":login, + "text":link(&bridge), "documents":[ + {"url":bridge,"text":link(&format!("{a}/returned"))}, + {"url":format!("{a}/returned"),"text":json!({"collections":[{"id":"roads", "links":[{"rel":"items", "type":"application/geo+json", "href":items}]}]}).to_string()} + ]}); + let plan = s.run("service.groupPlan", ¶ms).unwrap(); + let feature = &plan["layers"][0]; + assert_eq!(feature.get("anonymous").and_then(Value::as_bool).unwrap_or(false), crossed); + let listed = s.run("service.layers", ¶ms).unwrap(); + assert_eq!(listed["layers"][0].get("anonymous").and_then(Value::as_bool).unwrap_or(false), crossed); + params["responses"] = json!([{"url":feature["offer"]["url"], "text":r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[1,2]},"properties":{}}]}"#}]); + s.run("service.import", ¶ms).unwrap(); + let layer = &s.map().layers[0]; + assert_eq!(layer.anonymous, crossed); + assert_eq!(layer.connection.is_empty(), crossed); + assert_eq!(header(layer.dataset.source.as_deref().unwrap()).is_none(), crossed || !authenticated); + assert_eq!(layer.dataset.features.len(), 1); + params["group"] = json!(s.map().service_groups[0].id); + s.run("service.refresh", ¶ms).unwrap(); + let layer = &s.map().layers[0]; + assert_eq!(layer.anonymous, crossed); + assert_eq!(layer.connection.is_empty(), crossed); + assert_eq!(header(layer.dataset.source.as_deref().unwrap()).is_none(), crossed || !authenticated); + let saved = s.project.saved_text(None).unwrap().0; + s.run("project.openText", &json!({"text":saved})).unwrap(); + let layer = &s.map().layers[0]; + assert_eq!(layer.anonymous, crossed); + assert_eq!(layer.connection.is_empty(), crossed); + } +} diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 4751f0f18..01fcc8774 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -1209,7 +1209,7 @@ impl SourceLayer { } else if let Some(d) = duckdb && let Some(p) = &d.database { - out.push((R::DuckDbDatabase, p.as_str(), false, paths::is_local(p))); + out.push((R::DuckDbDatabase, p.as_str(), true, paths::is_local(p))); } if let Some(pg) = postgis { out.push((R::PostgisDatabase, pg.database.as_str(), true, false)); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index b961aca7f..0e2feffef 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -4390,12 +4390,15 @@ impl RustGisApp { } /// Refuses answers from a replaced project and reports the retired read without starting follow-up work. - fn service_read_current(&mut self, opened: u64, address: &str) -> bool { + fn service_read_current(&mut self, opened: u64, address: &str, pending: Option<&str>) -> bool { if opened == self.session.opened { return true; } let url = serviceui::address_in_message(address); let why = i18n::fmt("{url}: its project was closed while it was read, so it wasn't added", &[("url", &url)]); + if pending.is_some() && self.dialogs.info.as_deref() == pending { + self.dialogs.info = None; + } self.session.notify(why, false); false } @@ -4409,7 +4412,13 @@ impl RustGisApp { for step in std::mem::take(&mut self.ui.service_steps) { match step { ServiceStep::Tiles { mut resolve, dialog, origin, opened } => { - if !self.service_read_current(opened, &resolve.doc) { + let pending = dialog.then(|| { + i18n::fmt( + "Reading the tileset of {name}", + &[("name", resolve.params.get("name").and_then(Value::as_str).unwrap_or(""))], + ) + }); + if !self.service_read_current(opened, &resolve.doc, pending.as_deref()) { continue; } let Some(r) = resolve.slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { @@ -4468,7 +4477,9 @@ impl RustGisApp { self.map.invalidate(); } ServiceStep::Coverage { mut params, slot, dialog, origin, opened } => { - if !self.service_read_current(opened, params.get("url").and_then(Value::as_str).unwrap_or("")) { + let pending = dialog + .then(|| i18n::fmt("Describing {name}", &[("name", params.get("name").and_then(Value::as_str).unwrap_or(""))])); + if !self.service_read_current(opened, params.get("url").and_then(Value::as_str).unwrap_or(""), pending.as_deref()) { continue; } let Some(r) = slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { @@ -5581,11 +5592,54 @@ mod download_service_tests { assert!(app.ui.service_steps.is_empty()); let notices = app.session.notices.iter().filter(|n| n.text.contains("address(es) the service listed")).count(); assert_eq!(notices, count); - assert!(app.session.map().tiles.iter().all(|t| !t.url.contains("RGSYNTH_"))); + assert_eq!(app.session.map().tiles.len(), 1); + assert_eq!(app.session.map().tiles[0].url, "https://elsewhere.invalid/{z}/{y}/{x}?collections=roads"); assert!(app.dialogs.error.is_none(), "{:?}", app.dialogs.error); } } + #[test] + fn retired_service_steps_clear_only_their_own_pending_dialog() { + for coverage in [false, true] { + for newer in [false, true] { + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + let opened = app.session.opened; + let params = json!({"url":"https://retired.invalid/service", "name":"Old read"}); + let slot = std::sync::Arc::new(std::sync::Mutex::new(None)); + let step = if coverage { + super::ServiceStep::Coverage { params, slot: slot.clone(), dialog: true, origin: None, opened } + } else { + super::ServiceStep::Tiles { + resolve: super::TilesResolve { + params, + list: "https://retired.invalid/list".into(), + doc: "https://retired.invalid/list".into(), + crossed_origin: false, + read: 0, + withheld_addresses: 0, + vector: false, + fallback: "https://retired.invalid/{z}/{y}/{x}".into(), + slot: slot.clone(), + }, + dialog: true, + origin: None, + opened, + } + }; + app.ui.service_steps.push(step); + app.session.run("project.new", &json!({})).unwrap(); + let pending = if coverage { "Describing Old read" } else { "Reading the tileset of Old read" }; + app.dialogs.info = Some(if newer { "New operation" } else { pending }.into()); + *slot.lock().unwrap() = Some(Ok(b"late completion".to_vec())); + app.poll_service_steps(); + assert!(app.ui.service_steps.is_empty()); + assert!(app.session.map().tiles.is_empty() && app.session.map().layers.is_empty()); + assert!(app.session.notices.iter().any(|n| n.text.contains("its project was closed while it was read"))); + assert_eq!(app.dialogs.info.as_deref(), if newer { Some("New operation") } else { None }); + } + } + } + #[test] fn rejected_download_does_not_announce_success_or_rename_project() { let mut app = RustGisApp::with_sample() diff --git a/crates/ui-egui/src/serviceui.rs b/crates/ui-egui/src/serviceui.rs index 6247ac1f2..ca641ceb4 100644 --- a/crates/ui-egui/src/serviceui.rs +++ b/crates/ui-egui/src/serviceui.rs @@ -117,7 +117,13 @@ pub(crate) fn add_layer_from( // The layer's bounds go along, for its features' axes (#1053). let bounds = l.get("bbox").cloned().unwrap_or_default(); let mut params = json!({"url": items, "name": title, "service": true, "bounds": bounds}); - if let Some(connection) = connection { + let anonymous = l.get("anonymous").and_then(Value::as_bool) == Some(true) + || rustgis_engine::render::rustgis_tiles::logins::origin(&src.url) + != rustgis_engine::render::rustgis_tiles::logins::origin(items); + if anonymous { + params["url"] = json!(rustgis_engine::data::login_address::inner(items)); + params["anonymous"] = json!(true); + } else if let Some(connection) = connection { params["connection"] = json!(connection); } // An area asked for keeps the view, even on an empty map, as `service.add` does. @@ -180,6 +186,7 @@ pub(crate) fn add_layer_from( } } +/// Starts grouped discovery under its original request, retaining the import target and byte budget. pub fn start(app: &mut RustGisApp, p: &Value, refresh: bool) -> Result<(), String> { if !p.is_object() { return Err(tl!("Give the service's web address").into()); @@ -230,7 +237,7 @@ pub fn start(app: &mut RustGisApp, p: &Value, refresh: bool) -> Result<(), Strin let url = task.params.get("url").and_then(Value::as_str).unwrap_or_default(); let kind = task.params.get("type").and_then(Value::as_str); let caps = rustgis_engine::data::ogc::capabilities_url(url, kind); - let caps = match task.request(app, &caps) { + let caps = match task.request(&caps) { Ok(caps) => caps, Err(e) => { app.session.download_failed(task.token); @@ -247,14 +254,18 @@ pub fn start(app: &mut RustGisApp, p: &Value, refresh: bool) -> Result<(), Strin impl Import { /// Anonymous service provenance forbids selecting a login, including for same-origin links. - fn request(&self, app: &mut RustGisApp, url: &str) -> Result { + fn request(&self, url: &str) -> Result { if self.params.get("anonymous").and_then(Value::as_bool) == Some(true) { return Ok(rustgis_engine::data::login_address::inner(url).to_string()); } let base = self.params.get("url").and_then(Value::as_str).unwrap_or_default(); - request_from(app, self.connection.as_ref().and_then(Value::as_str), base, url) + // The plan already resolved this exact destination and its login (or absence). + // Re-selecting from the original saved connection would undo a foreign hop. + rustgis_engine::render::rustgis_tiles::address::check(url, Some(base)).map_err(|e| crate::connectionui::translated(&e))?; + Ok(url.to_string()) } + /// Consumes the engine plan verbatim: foreign-hop anonymity cannot reselect a login. fn discover(&mut self, app: &mut RustGisApp) -> Result<(), String> { let listing = app.session.run("service.groupPlan", &self.params)?; if listing.get("request").and_then(|p| p.get("anonymous")).and_then(Value::as_bool) == Some(true) { @@ -265,7 +276,7 @@ impl Import { self.connection = None; } if let Some(next) = listing.get("next").and_then(Value::as_str) { - let next = self.request(app, next)?; + let next = self.request(next)?; self.slot = fetch_into(&next); self.document = Some(next); } else { @@ -276,20 +287,22 @@ impl Import { .flatten() .filter_map(|l| l.get("offer").and_then(|o| o.get("url")).and_then(Value::as_str).map(str::to_string)) .collect(); - self.next(app)?; + self.next()?; } Ok(()) } - fn next(&mut self, app: &mut RustGisApp) -> Result<(), String> { + /// Fetches one exact planned collection within the remaining shared byte budget. + fn next(&mut self) -> Result<(), String> { self.current = self.remaining.pop_front(); if let Some(url) = &self.current { - let url = self.request(app, url)?; + let url = self.request(url)?; self.slot = fetch_into_limited(&url, rustgis_engine::cmd::service::MAX_GROUP_BYTES.saturating_sub(self.bytes)); } Ok(()) } + /// Accumulates supplied documents or feature responses before the engine import/refresh. fn landed(&mut self, app: &mut RustGisApp, bytes: Vec) -> Result<(), String> { if let Some(document) = self.document.take() { let text = crate::service_text(bytes).map_err(|_| tl!("The service's answer isn't UTF-8 text").to_string())?; @@ -309,7 +322,7 @@ impl Import { } let text = String::from_utf8(bytes).map_err(|_| tl!("The service's answer isn't UTF-8 text").to_string())?; self.responses.push(json!({"url":self.current.take(),"text":text})); - self.next(app)?; + self.next()?; Ok(()) } } diff --git a/docs/python.md b/docs/python.md index 52d15915a..c367c7b86 100644 --- a/docs/python.md +++ b/docs/python.md @@ -139,9 +139,11 @@ The project, with the data that travels with it, goes to that app through the br (`postMessage`); nothing goes to a server. The app runs in your browser, so with the default address your data stays on your machine. Point `url=` only at a copy of the app you trust. -The widget receives public addresses and cleaned executable parameters. Credentials held by -Python are not sent to it, so protected layers can remain empty there; a warning counts affected -layer addresses. When the widget sends a project back, Python can retain credentials only for +The widget receives public addresses and cleaned executable parameters. Recognized credentials +in supported fields are removed, so protected layers can remain empty there; a warning counts +affected layer addresses. User data, foreign notebook source and opaque parameters are kept +unchanged and may contain credentials: review them before sharing the project with a widget. +When the widget sends a project back, Python can retain credentials only for unchanged, uniquely matched sources from its last exchange that still match its current project. Changing or replacing a source can require signing in again in Python. Exchange notices are available through `m.notices()` on the `Map` or `MapWidget`. diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index 3efe327be..bea32fc55 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -32,12 +32,12 @@ The browser app doesn't ask yet; it warns before you leave a page with unsaved c default), RustGIS keeps your changes as a [recovery copy](#autosave-and-recovery) and offers them back the next time it starts; with autosave off, they are lost, so save before you quit. -Saved layer and video addresses, and recognized task, model, notebook and history parameters, -omit passwords, access keys, tokens and signatures. Notices count what was removed or could not +Supported layer and video address fields, and recognized task, model, notebook and history +parameters, omit recognized passwords, access keys, tokens and signatures. Notices count what was removed or could not be checked. The open session keeps its credentials, but a saved workflow or protected source may need them again when reopened. User notes, feature data and opaque content are preserved; see [Sharing projects and generated items](https://github.com/opengeos/RustGIS/blob/main/SECURITY.md#sharing-projects-and-generated-items) -before sharing a file. +before sharing a file. Review addresses before sharing too: an unrecognized credential can remain unchanged. ### Autosave and recovery From d533b89d3a34bff694cf3e2e1b41205a989988d8 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 7 Nov 2026 01:39:06 -0600 Subject: [PATCH 3/4] fix(security): close identifier and generated-output leaks --- crates/assistant/src/audit.rs | 55 ++++-- crates/data/src/address.rs | 14 ++ crates/data/src/diagnostics.rs | 12 ++ crates/engine/src/durable.rs | 50 +++--- crates/engine/src/lib.rs | 23 ++- crates/engine/src/project.rs | 32 ++-- crates/engine/src/project_log.rs | 7 + crates/engine/src/roles.rs | 52 +++++- crates/engine/src/sources.rs | 3 +- crates/engine/src/tests/credential_output.rs | 177 +++++++++++++++++++ crates/engine/src/tests/service_logins.rs | 13 +- crates/tiles/src/cache.rs | 43 +++-- crates/tiles/src/logins.rs | 3 + 13 files changed, 393 insertions(+), 91 deletions(-) diff --git a/crates/assistant/src/audit.rs b/crates/assistant/src/audit.rs index 3f9d40f82..952191ed3 100644 --- a/crates/assistant/src/audit.rs +++ b/crates/assistant/src/audit.rs @@ -150,7 +150,11 @@ fn normalized_args(tool: &str, arguments: &Value) -> Value { }; let key = if tool == "run_command" { "command" } else { "tool" }; let mut out = serde_json::Map::new(); - out.insert(key.into(), redact(arguments.get(key).cloned().unwrap_or(Value::Null), 0)); + let id = match arguments.get(key) { + Some(Value::String(id)) => json!(rustgis_engine::log_identifier(id)), + other => redact(other.cloned().unwrap_or(Value::Null), 0), + }; + out.insert(key.into(), id); out.insert("params".into(), params); Value::Object(out) } @@ -233,15 +237,22 @@ fn redact_script(script: &str) -> String { let output = script .lines() .take(rustgis_engine::cmd::workflows::MAX_SCRIPT_LINES) - .map(|line| match rustgis_engine::script_line(line) { - None => redact_string(line.trim()), - Some(Ok((id, params))) => { - let id = redact_string(&id); - if params.is_null() { id } else { format!("{id} {}", rustgis_engine::log_params(¶ms).into_log_value()) } - } - Some(Err(_)) => { - let id = line.trim().split_once(char::is_whitespace).map_or(line.trim(), |(id, _)| id).trim_end_matches('='); - format!("{} [parameters not JSON; withheld]", redact_string(id)) + .map(|line| { + let line = line.trim(); + // Display only: a malformed identifier may carry JSON glued after `=`. It must + // not be copied as an id; execution still uses the shared parser unchanged. + let head = line.split(|c: char| c.is_whitespace() || c == '=').next().unwrap_or(line); + let glued = line + .get(head.len()..) + .and_then(|r| r.strip_prefix('=')) + .is_some_and(|r| !r.is_empty() && !r.starts_with(char::is_whitespace)); + match rustgis_engine::script_line(line) { + None => redact_string(line), + Some(Ok((id, params))) if !glued => { + let id = rustgis_engine::log_identifier(&id); + if params.is_null() { id } else { format!("{id} {}", rustgis_engine::log_params(¶ms).into_log_value()) } + } + Some(_) => format!("{} [parameters not JSON; withheld]", rustgis_engine::log_identifier(head)), } }) .collect::>() @@ -308,6 +319,30 @@ mod tests { #[test] fn audit_logs_the_executed_envelope_and_shared_script_grammar() { + for line in [ + "sql.connect={\"password\":\"RGSYNTH_glued_audit\"}", + "sql.connect={\"password\":\"RGSYNTH_glued_audit\"} junk", + "sql.connect={\"password\":\"RGSYNTH_glued_audit\"", + ] { + let output = redact_script(line); + assert!(!output.contains("RGSYNTH_glued_audit"), "{output}"); + } + for id in [ + "sql.connect={\"password\":\"RGSYNTH_glued_audit\"}", + "sql.connect {\"password\":\"RGSYNTH_glued_audit\"}", + "sql.connect:{\"password\":\"RGSYNTH_glued_audit\"}", + "sql.connect:password:RGSYNTH_glued_audit", + "sql.connect:token:RGSYNTH_glued_audit", + "password:RGSYNTH_glued_audit", + ] { + for tool in ["run_command", "run_tool"] { + let key = if tool == "run_command" { "command" } else { "tool" }; + let mut args = json!({"params":null}); + args[key] = json!(id); + let logged = normalized_args(tool, &args); + assert!(!logged.to_string().contains("RGSYNTH_glued_audit"), "{tool}: {logged}"); + } + } let params = json!({"url": "https://h.invalid/a?%74oken=RGSYNTH_audit_20261103", "pass%77ord": "RGSYNTH_option_20261103", "layer": "roads"}); for tool in ["run_command", "run_tool"] { diff --git a/crates/data/src/address.rs b/crates/data/src/address.rs index 0eba9a766..c85d7ea1e 100644 --- a/crates/data/src/address.rs +++ b/crates/data/src/address.rs @@ -331,6 +331,11 @@ pub fn public_web_address(address: &str) -> Public { if public == text { Public::Unchanged } else { Public::Changed { address: public, removed } } } +/// A bare supported scheme is a prose example with no authority or credential to remove. +pub(crate) fn scheme_example(text: &str) -> bool { + ["http://", "https://", "s3://"].iter().any(|scheme| text.eq_ignore_ascii_case(scheme)) +} + /// Cleans addresses in generated text and counts changed addresses. All spacing and all other /// text stay byte-for-byte. Never apply this to user-authored text or dataset values. pub fn scrub_addresses(text: &str) -> (Cow<'_, str>, u32) { @@ -365,6 +370,10 @@ pub fn scrub_addresses(text: &str) -> (Cow<'_, str>, u32) { .len() .max(scheme_at + 1); let address = word.get(begin..end).unwrap_or(""); + // A bare scheme in diagnostic prose has no address or credentials to remove. + if scheme_example(address) { + continue; + } let public = public_web_address(address); if matches!(public, Public::Unchanged) { continue; @@ -462,6 +471,8 @@ pub struct ReviewedScope { pub directory: &'static str, pub whole_origin: bool, } +/// No deployment has been authorized for full-query inheritance. Keep this list empty until +/// that review exists; the explicit-list tests exercise the same deny boundary. pub const F2_SCOPES: &[ReviewedScope] = &[]; #[derive(Debug, PartialEq, Eq)] @@ -620,6 +631,9 @@ mod tests { ("[s3:bucket/x?token=RGSYNTH_0]", "[s3:]", 1), ("https:x", "https:", 1), ("s3:", "s3:", 1), + ("plain http://, confirm before sending", "plain http://, confirm before sending", 0), + ("use https:// for encryption", "use https:// for encryption", 0), + ("an S3:// address", "an S3:// address", 0), ("read https:/h.invalid/x then https://h.invalid/b?layer=a%20b", "read https: then https://h.invalid/b?layer=a%20b", 1), ( "http://h.invalid/a https://h.invalid/b?layer=a%20b s3://bucket/x", diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 9734088c9..7da31f998 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -639,6 +639,12 @@ pub fn redact(text: &str) -> String { if after.starts_with("://") { let url_end = rest.find(|c: char| c.is_whitespace() || matches!(c, '"' | '\'' | '<' | '>' | '`')).unwrap_or(rest.len()); let address = rest.get(..url_end).unwrap_or(""); + if crate::address::scheme_example(address.trim_end_matches([':', ',', ';', '.', ')', ']'])) { + out.push_str(address); + i += url_end; + field_before = false; + continue; + } match url::Url::parse(address) { // Nothing to take out: the address stays as written, its punctuation too // ("https://example.org: timeout"), unless it holds control characters. @@ -1148,6 +1154,12 @@ mod tests { fn addresses_without_anything_to_take_out_stay_as_written() { assert_eq!(redact("Could not reach https://example.org: timeout"), "Could not reach https://example.org: timeout"); assert_eq!(redact("failed https://example.org/a/b, retrying"), "failed https://example.org/a/b, retrying"); + for text in ["plain http://, confirm before sending", "use https:// for encryption", "an S3:// address"] { + assert_eq!(redact(text), text); + } + for address in ["https://?token=RGSYNTH_example", "s3://#RGSYNTH_example", "https://u:RGSYNTH_example@"] { + assert!(!redact(address).contains("RGSYNTH_example"), "an incomplete address with content is still withheld"); + } } /// Taking user information out rewrites the address, and the punctuation that ended it in the diff --git a/crates/engine/src/durable.rs b/crates/engine/src/durable.rs index e1e4b88b1..939022141 100644 --- a/crates/engine/src/durable.rs +++ b/crates/engine/src/durable.rs @@ -113,11 +113,15 @@ pub(crate) enum Leaf { Layer(rustgis_map::LayerRole), Table, Task(usize), + TaskCommand(usize), ModelStep(usize), + ModelStepTool(usize), ModelNode(usize), - ModelLayer(usize), + ModelNodeTool(usize), HistoryParams, + HistoryId, HistoryCommand, + HistoryCommandId, Message(usize), CellSource(usize), CellOutput(usize), @@ -201,10 +205,8 @@ fn json_bytes(v: &Value) -> usize { pub(crate) trait Durable { /// Preparation is read-only, exhaustive over owning fields, and offers each leaf once. - /// Offers classified whole fields once, before any live mutation. fn prepare(&self, p: &mut Prepare<'_>); /// Resolves recorded indices only, through plain owned fields, without allocation or policy. - /// Resolves a prepared location through owned fields, without allocation or policy. fn field_mut(&mut self, owners: &[Owner], leaf: &Leaf) -> Option>; } @@ -290,32 +292,33 @@ impl<'a> Prepare<'a> { } } - /// Prepares a command's replayable parameters using explicit data/configuration roles. - pub(crate) fn command(&mut self, leaf: Leaf, id: &str, value: &Value) { - let (clean, report) = crate::roles::command(id, value, self.depth); - let clean = match clean { - Some(clean) => Some(crate::project::public_login_params(&clean).unwrap_or(clean)), - None => crate::project::public_login_params(value), - }; - self.report.merge(&report); - if let Some(clean) = clean { - self.report.steps = self.report.steps.saturating_add(1); - self.edit(leaf, Payload::Json(clean), Kind::Credential); - } + /// Prepares both fields of one replayable command, counting the owning step once. + pub(crate) fn command(&mut self, leaf: Leaf, id_leaf: Leaf, id: &str, value: &Value) { + self.executable((leaf, id_leaf), id, value, crate::roles::command(id, value, self.depth)); + } + + /// Model and history tool params use the same common roles as `gp.run` params. + pub(crate) fn tool(&mut self, leaf: Leaf, id_leaf: Leaf, id: &str, value: &Value) { + self.executable((leaf, id_leaf), id, value, crate::roles::tool(id, value, self.depth)); } - /// Model and history tool params have the same roles as the corresponding `gp.run` params. - pub(crate) fn tool(&mut self, leaf: Leaf, id: &str, value: &Value) { - let (clean, report) = crate::roles::tool(id, value, self.depth); + /// Journals the identifier and params independently while reporting their shared owner. + fn executable(&mut self, (leaf, id_leaf): (Leaf, Leaf), id: &str, value: &Value, (clean, report): (Option, Report)) { let clean = match clean { Some(clean) => Some(crate::project::public_login_params(&clean).unwrap_or(clean)), None => crate::project::public_login_params(value), }; self.report.merge(&report); - if let Some(clean) = clean { + let clean_id = crate::roles::identifier(id, &mut self.report); + if clean.is_some() || clean_id.is_some() { self.report.steps = self.report.steps.saturating_add(1); + } + if let Some(clean) = clean { self.edit(leaf, Payload::Json(clean), Kind::Credential); } + if let Some(clean) = clean_id { + self.edit(id_leaf, Payload::Text(clean), Kind::Credential); + } } /// Rewrites only changed command lines; comments, data and foreign-language cells stay exact. @@ -574,7 +577,8 @@ fn public_address_at(address: &str, depth: usize) -> Kept { address::{Public, public_web_address}, s3_address, }; - if let Some((public, _)) = rustgis_data::source::myconn::split_password(address) { + if let Some((mut public, _)) = rustgis_data::source::myconn::split_password(address) { + rustgis_data::address::redact_query(&mut public); return if public == address { Kept::Unchanged } else { Kept::Changed(public) }; } if let Some((public, _)) = rustgis_data::source::msconn::split_password(address) { @@ -683,11 +687,15 @@ pub(crate) fn changed_executables(old: &mut crate::Project, incoming: &mut crate || !matches!( edit.at.leaf, Leaf::Task(_) + | Leaf::TaskCommand(_) | Leaf::ModelStep(_) + | Leaf::ModelStepTool(_) | Leaf::ModelNode(_) - | Leaf::ModelLayer(_) + | Leaf::ModelNodeTool(_) | Leaf::HistoryParams + | Leaf::HistoryId | Leaf::HistoryCommand + | Leaf::HistoryCommandId | Leaf::CellSource(_) ) { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 0ed79a97e..7941d0f85 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -20,7 +20,7 @@ pub(crate) use roles::{Document, ParamRole, Role}; mod parameters; pub mod project_log; pub use parameters::{params_envelope, script_line}; -pub use project_log::{LogCopy, log_params}; +pub use project_log::{LogCopy, log_identifier, log_params}; pub mod s3_connections; pub mod s3_keys; #[cfg(test)] @@ -1466,7 +1466,12 @@ impl Session { } } + /// Cleans every returned error, including early refusal and plugin paths. fn run_command(&mut self, id: &str, params: &Value) -> CmdResult { + self.run_command_inner(id, params).map_err(|e| durable::generated_text(&e)) + } + + fn run_command_inner(&mut self, id: &str, params: &Value) -> CmdResult { // The merge key asked for ([`Session::merge_undo`]) goes to this command if it runs, and is // gone if it's refused first, so it can't merge a later command into the last step. let merge = if self.depth == 0 { self.merge.take() } else { None }; @@ -1478,7 +1483,7 @@ impl Session { self.merge = merge; return self.run_plugin(id, params); } - let Some(&spec) = self.registry.get(id) else { return Err(format!("unknown command '{id}'")) }; + let Some(&spec) = self.registry.get(id) else { return Err(format!("unknown command '{}'", project_log::log_identifier(id))) }; // Keys the command does not read: said, so a script or an agent can correct itself. if let (Value::Object(given), Some(known)) = (params, spec.param_keys()) { // Bounded: the keys come from the caller (the first few, each clipped). @@ -1545,8 +1550,7 @@ impl Session { { self.sources.note(src); } - // An error never quotes a web-service connection's password or key. - res.map_err(|e| render::rustgis_tiles::logins::scrub(&e)) + res } /// Retires only this consumed reservation, preserving unrelated pending imports in history. @@ -2026,7 +2030,7 @@ impl Session { /// redacted: what a notice or history record may show. Login values are known only here, so /// a failure redacted by [`Session::notify_redacted`]'s callers goes through this first. pub fn redacted(text: &str) -> rustgis_data::diagnostics::Redacted { - rustgis_data::diagnostics::Redacted::new(&render::rustgis_tiles::logins::scrub(text)) + rustgis_data::diagnostics::Redacted::new(&durable::generated_text(text)) } /// [`Session::notify`] for text already scrubbed and redacted by [`Session::redacted`] (a @@ -2521,9 +2525,9 @@ impl durable::Durable for GpRecord { use durable::Leaf; let Self { tool, id, params, command, at: _, ok: _, messages, millis: _, outputs: _ } = self; let tool = if id.is_empty() { cmd::gp::tool_id(tool).unwrap_or("") } else { id }; - p.tool(Leaf::HistoryParams, tool, params); + p.tool(Leaf::HistoryParams, Leaf::HistoryId, tool, params); if let Some((id, params)) = command { - p.command(Leaf::HistoryCommand, id, params); + p.command(Leaf::HistoryCommand, Leaf::HistoryCommandId, id, params); } for (i, message) in messages.iter().enumerate() { p.generated(Leaf::Message(i), message); @@ -2536,7 +2540,9 @@ impl durable::Durable for GpRecord { } match leaf { Leaf::HistoryParams => Some(FieldMut::Json(&mut self.params)), + Leaf::HistoryId => Some(FieldMut::Text(&mut self.id)), Leaf::HistoryCommand => Some(FieldMut::Json(&mut self.command.as_mut()?.1)), + Leaf::HistoryCommandId => Some(FieldMut::Text(&mut self.command.as_mut()?.0)), Leaf::Message(i) => self.messages.get_mut(*i).map(FieldMut::Text), _ => None, } @@ -2547,8 +2553,7 @@ impl Session { /// The single history production seam: retained generated messages never keep address keys. pub(crate) fn push_gp_record(&mut self, mut record: GpRecord) { for message in &mut record.messages { - let scrubbed = render::rustgis_tiles::logins::scrub(message); - *message = rustgis_data::address::scrub_addresses(&scrubbed).0.into_owned(); + *message = durable::generated_text(message); } self.project.gp_history.push(record); } diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 08bb586e2..8c7a4c5da 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -650,10 +650,7 @@ impl Project { if let Some(graph) = &mut model.graph { for node in &mut graph.nodes { recorded_addresses(&mut node.params, &mut addresses); - // A node's input may be an address rather than a layer's name. - if let Some(layer) = &mut node.layer { - addresses.push(layer); - } + // node.layer selects a name, including names that resemble runtime URLs. } } } @@ -1287,7 +1284,7 @@ impl Durable for Task { fn prepare(&self, p: &mut Prepare<'_>) { let Self { name: _, description: _, steps } = self; for (i, TaskStep { title: _, instructions: _, command, params }) in steps.iter().enumerate() { - p.command(Leaf::Task(i), command, params); + p.command(Leaf::Task(i), Leaf::TaskCommand(i), command, params); } } @@ -1296,8 +1293,11 @@ impl Durable for Task { if !owners.is_empty() { return None; } - let Leaf::Task(i) = leaf else { return None }; - Some(FieldMut::Json(&mut self.steps.get_mut(*i)?.params)) + match leaf { + Leaf::Task(i) => Some(FieldMut::Json(&mut self.steps.get_mut(*i)?.params)), + Leaf::TaskCommand(i) => Some(FieldMut::Text(&mut self.steps.get_mut(*i)?.command)), + _ => None, + } } } @@ -1306,7 +1306,7 @@ impl Durable for Model { fn prepare(&self, p: &mut Prepare<'_>) { let Self { name: _, steps, graph } = self; for (i, ModelStep { tool, params }) in steps.iter().enumerate() { - p.tool(Leaf::ModelStep(i), tool, params); + p.tool(Leaf::ModelStep(i), Leaf::ModelStepTool(i), tool, params); } if let Some(g) = graph { p.child(Owner::Graph, g); @@ -1321,8 +1321,11 @@ impl Durable for Model { if !owners.is_empty() { return None; } - let Leaf::ModelStep(i) = leaf else { return None }; - Some(FieldMut::Json(&mut self.steps.get_mut(*i)?.params)) + match leaf { + Leaf::ModelStep(i) => Some(FieldMut::Json(&mut self.steps.get_mut(*i)?.params)), + Leaf::ModelStepTool(i) => Some(FieldMut::Text(&mut self.steps.get_mut(*i)?.tool)), + _ => None, + } } } @@ -1330,11 +1333,8 @@ impl Durable for ModelGraph { /// Offers classified whole fields once, before any live mutation. fn prepare(&self, p: &mut Prepare<'_>) { let Self { nodes, edges: _ } = self; - for (i, ModelNode { id: _, kind: _, x: _, y: _, layer, tool, params, name: _ }) in nodes.iter().enumerate() { - p.tool(Leaf::ModelNode(i), tool.as_deref().unwrap_or(""), params); - if let Some(layer) = layer { - p.string(Leaf::ModelLayer(i), layer, true, false); - } + for (i, ModelNode { id: _, kind: _, x: _, y: _, layer: _, tool, params, name: _ }) in nodes.iter().enumerate() { + p.tool(Leaf::ModelNode(i), Leaf::ModelNodeTool(i), tool.as_deref().unwrap_or(""), params); } } @@ -1345,7 +1345,7 @@ impl Durable for ModelGraph { } match leaf { Leaf::ModelNode(i) => Some(FieldMut::Json(&mut self.nodes.get_mut(*i)?.params)), - Leaf::ModelLayer(i) => self.nodes.get_mut(*i)?.layer.as_mut().map(FieldMut::Text), + Leaf::ModelNodeTool(i) => self.nodes.get_mut(*i)?.tool.as_mut().map(FieldMut::Text), _ => None, } } diff --git a/crates/engine/src/project_log.rs b/crates/engine/src/project_log.rs index 7fbead77f..e1114e025 100644 --- a/crates/engine/src/project_log.rs +++ b/crates/engine/src/project_log.rs @@ -22,6 +22,13 @@ pub fn log_params(params: &Value) -> LogCopy { LogCopy(kept_params(params)) } +/// Cleans a structural command/tool identifier for display and audit output only. +/// The original identifier remains the sole input to lookup, execution and approval. +pub fn log_identifier(id: &str) -> String { + let public = crate::roles::identifier(id, &mut crate::durable::Report::default()); + crate::durable::generated_text(public.as_deref().unwrap_or(id)) +} + /// Now, in milliseconds since 1970 (UTC). pub(crate) fn now_ms() -> u64 { web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0) diff --git a/crates/engine/src/roles.rs b/crates/engine/src/roles.rs index 3d7c069fa..cc3897628 100644 --- a/crates/engine/src/roles.rs +++ b/crates/engine/src/roles.rs @@ -119,6 +119,31 @@ pub(crate) fn tool(id: &str, params: &Value, depth: usize) -> (Option, Re (changed, report) } +/// Identifiers are structural executable fields, not layer names or user data. Recognized +/// addresses are cleaned; credential-bearing malformed payloads are withheld as a whole. +pub(crate) fn identifier(id: &str, report: &mut Report) -> Option { + // A colon-separated credential key/value is payload; a qualified plugin/tool id alone is not. + let colon_value = id + .split(':') + .zip(id.split(':').skip(1)) + .take(65_536) + .any(|(key, value)| !value.is_empty() && rustgis_data::address::secret_key(key)); + match durable::public_address(id) { + Kept::Changed(public) => Some(public), + Kept::Withheld(prefix) if prefix != id => { + report.withheld = report.withheld.saturating_add(1); + Some(prefix.into()) + } + _ if (colon_value || id.contains(['=', '{', '[', '"']) || id.contains(char::is_whitespace)) + && suspicious(&Value::String(id.into())) => + { + report.withheld = report.withheld.saturating_add(1); + Some("[command withheld]".into()) + } + _ => None, + } +} + /// Lazily copies only changed owning parameter objects. Configuration recursion is bounded. fn object(id: &str, value: &Value, depth: usize, config_depth: usize, registered: bool, report: &mut Report) -> Option { if value.is_null() { @@ -306,7 +331,7 @@ impl Durable for VecTask { /// Offers classified whole fields once, before any live mutation. fn prepare(&self, p: &mut durable::Prepare<'_>) { for (i, st) in self.0.iter().enumerate() { - p.command(durable::Leaf::Task(i), &st.command, &st.params); + p.command(durable::Leaf::Task(i), durable::Leaf::TaskCommand(i), &st.command, &st.params); } } @@ -315,8 +340,11 @@ impl Durable for VecTask { if !owners.is_empty() { return None; } - let durable::Leaf::Task(i) = leaf else { return None }; - Some(durable::FieldMut::Json(&mut self.0.get_mut(*i)?.params)) + match leaf { + durable::Leaf::Task(i) => Some(durable::FieldMut::Json(&mut self.0.get_mut(*i)?.params)), + durable::Leaf::TaskCommand(i) => Some(durable::FieldMut::Text(&mut self.0.get_mut(*i)?.command)), + _ => None, + } } } @@ -328,7 +356,7 @@ impl Durable for VecSteps { /// Offers classified whole fields once, before any live mutation. fn prepare(&self, p: &mut durable::Prepare<'_>) { for (i, st) in self.0.iter().enumerate() { - p.tool(durable::Leaf::ModelStep(i), &st.tool, &st.params); + p.tool(durable::Leaf::ModelStep(i), durable::Leaf::ModelStepTool(i), &st.tool, &st.params); } } @@ -337,8 +365,11 @@ impl Durable for VecSteps { if !owners.is_empty() { return None; } - let durable::Leaf::ModelStep(i) = leaf else { return None }; - Some(durable::FieldMut::Json(&mut self.0.get_mut(*i)?.params)) + match leaf { + durable::Leaf::ModelStep(i) => Some(durable::FieldMut::Json(&mut self.0.get_mut(*i)?.params)), + durable::Leaf::ModelStepTool(i) => Some(durable::FieldMut::Text(&mut self.0.get_mut(*i)?.tool)), + _ => None, + } } } @@ -418,14 +449,19 @@ pub(crate) fn script(source: &str, depth: usize) -> (Option, Report) { let replacement = match crate::script_line(line) { None => None, Some(Ok((id, params))) => { + let public_id = identifier(&id, &mut report); let (clean, r) = command(&id, ¶ms, depth); report.merge(&r); - clean.map(|params| format!("{id} {params}{ending}")) + match clean { + Some(params) => Some(format!("{} {params}{ending}", public_id.as_deref().unwrap_or(&id))), + None => public_id.map(|public| line.replacen(&id, &public, 1)), + } } Some(Err(_)) if suspicious(&Value::String(line.into())) => { report.withheld = report.withheld.saturating_add(1); let id = line.trim_start().split(char::is_whitespace).next().unwrap_or(""); - Some(format!("{id} {WITHHELD}{ending}")) + let public_id = identifier(id, &mut report); + Some(format!("{} {WITHHELD}{ending}", public_id.as_deref().unwrap_or(id))) } Some(Err(_)) => None, }; diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index a196e295d..e8315d6e6 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -365,7 +365,8 @@ pub fn redact(address: &str) -> String { { return address.to_string(); } - if let Some((public, _)) = rustgis_data::source::myconn::split_password(address) { + if let Some((mut public, _)) = rustgis_data::source::myconn::split_password(address) { + rustgis_data::address::redact_query(&mut public); return public; } if let Some((public, _)) = rustgis_data::source::msconn::split_password(address) { diff --git a/crates/engine/src/tests/credential_output.rs b/crates/engine/src/tests/credential_output.rs index 76f5c7bba..da66e7789 100644 --- a/crates/engine/src/tests/credential_output.rs +++ b/crates/engine/src/tests/credential_output.rs @@ -9,6 +9,183 @@ use rustgis_map::presentation::{Page, PageKind, Presentation}; const SECRET: &str = "RGSYNTH_output_20261103"; const DATA: &str = "RGSYNTH_data_20261103"; +/// Database labels and durable addresses share query-key classification, without changing +/// the executable connection string or dropping clean connection options. +#[test] +fn durable_database_query_aliases_match_public_labels() { + for scheme in ["mysql", "mariadb", "mssql", "postgresql"] { + for key in ["access_token", "subscription-key", "api_key"] { + let public = format!("{scheme}://reader@db.invalid/gis?ssl=true"); + let private = format!("{public}&{key}={SECRET}"); + let kept = match durable::public_address(&private) { + durable::Kept::Changed(s) => s, + _ => private.clone(), + }; + assert_eq!(kept, public, "durable {scheme}/{key}"); + assert_eq!(crate::sources::redact(&private), public, "public label {scheme}/{key}"); + assert!(matches!(durable::public_address(&public), durable::Kept::Unchanged)); + assert_eq!(crate::sources::redact(&public), public); + } + } +} + +/// Graph input fields select existing names; source-address cleaning must not rename those +/// references when saved text is opened without this process's credential restore memory. +#[test] +fn durable_model_input_names_survive_a_cold_round_trip() { + let bound = rustgis_data::login_address::bind("0123456789abcdef", "https://example.invalid", "https://example.invalid/roads").unwrap(); + for name in ["https://example.invalid/roads#north", bound.as_str()] { + let mut s = world(); + s.map_mut().layers[0].name = name.into(); + let graph = ModelGraph { + nodes: vec![ModelNode { + id: "input1".into(), + kind: crate::project::ModelNodeKind::Input, + layer: Some(name.into()), + ..Default::default() + }], + edges: Vec::new(), + }; + s.project.models.push(Model { name: "Names".into(), steps: Vec::new(), graph: Some(graph) }); + let (text, _) = s.project.saved_text(None).unwrap(); + for cold in [Project::from_json(&text).unwrap(), Project::from_import_json(&text, None).unwrap()] { + let model = cold.models.last().unwrap().graph.as_ref().unwrap(); + let selector = model.nodes[0].layer.as_deref().unwrap(); + assert_eq!(selector, name, "a graph input selects a name, even when it resembles an address or runtime wrapper"); + assert!(cold.map().layer_by_name(selector).is_some(), "the opened graph still resolves its real input"); + } + } +} + +/// Address parsing must precede active-pattern masking at every generated history/error +/// boundary, even when the active key is part of the URL scheme itself. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn generated_errors_and_history_clean_addresses_before_active_patterns() { + if isolate_process_test("tests::credential_output::generated_errors_and_history_clean_addresses_before_active_patterns") { + return; + } + use crate::render::rustgis_tiles::logins; + let binding = logins::activate( + "https://history-collision.invalid", + "synthetic-history-collision", + logins::Method::Header("X-Key".into()), + "https", + false, + ) + .unwrap(); + let raw = format!("failed at https://h.invalid/a?token={SECRET}&layer=roads"); + let mut s = world(); + s.push_gp_record(GpRecord { + tool: "Unknown".into(), + id: "unknown".into(), + params: Value::Null, + command: None, + at: 0, + ok: false, + messages: vec![raw.clone()], + millis: 0, + outputs: Vec::new(), + }); + s.notify(&raw, true); + let error = s.run("layer.rename", &json!({"layer":format!("https://h.invalid/a?token={SECRET}"),"name":"unused"})).unwrap_err(); + let history = s.project.gp_history.last().unwrap().messages[0].clone(); + let notice = s.notices.last().unwrap().text.clone(); + assert!(logins::deactivate(&binding)); + assert!(!history.contains(SECRET), "history: {history}"); + assert!(!notice.contains(SECRET), "notice: {notice}"); + assert!(!error.contains(SECRET), "command error: {error}"); + for id in [ + format!("https://h.invalid/a?token={SECRET}"), + format!("sql.connect={{\"password\":\"{SECRET}\"}}"), + format!("sql.connect {{\"password\":\"{SECRET}\"}}"), + ] { + let error = s.run(&id, &Value::Null).unwrap_err(); + assert!(!error.contains(SECRET), "unknown command error: {error}"); + } + assert!(!s.project.saved_text(None).unwrap().0.contains(SECRET)); +} + +/// Invalid command/tool identifiers are executable slots too: an address there must not +/// bypass cleaning through a null parameter or a malformed script line. +#[test] +fn durable_identifier_addresses_are_cleaned_and_counted() { + let url = format!("https://identifier.invalid/a?token={SECRET}&layer=roads"); + let source = format!( + "# clean comment\r\n{url}\r\n{url} {{broken\nsql.connect={{\"password\":\"{SECRET}\"}}\nsql.connect={{\"password\":\"{SECRET}\"}} junk\nmap.zoomIn {{}}\n" + ); + let mut project = Project::default(); + project.notebooks.push(Notebook { + name: "Identifiers".into(), + jupyter: None, + cells: vec![Cell { kind: CellKind::Code, source, output: None, error: false, jupyter: None }], + }); + project.tasks.push(Task { + name: "Identifiers".into(), + description: String::new(), + steps: [ + url.clone(), + format!("sql.connect {{\"password\":\"{SECRET}\"}}"), + format!("sql.connect:{{\"password\":\"{SECRET}\"}}"), + format!("sql.connect:password:{SECRET}"), + format!("sql.connect:token:{SECRET}"), + ] + .into_iter() + .map(|command| TaskStep { title: String::new(), instructions: String::new(), command, params: Value::Null }) + .collect(), + }); + project.models.push(Model { + name: "Identifiers".into(), + steps: vec![ModelStep { tool: url.clone(), params: Value::Null }], + graph: Some(ModelGraph { + nodes: vec![ModelNode { id: "tool1".into(), tool: Some(url.clone()), ..Default::default() }], + edges: Vec::new(), + }), + }); + let mut s = Session::new(project); + s.project.gp_history.push(GpRecord { + tool: "Unknown".into(), + id: url.clone(), + params: Value::Null, + command: Some((url, Value::Null)), + at: 0, + ok: false, + messages: Vec::new(), + millis: 0, + outputs: Vec::new(), + }); + let original = serde_json::to_value(&s.project).unwrap(); + let (text, report) = s.project.saved_text(None).unwrap(); + assert!(!text.contains(SECRET), "an identifier bypassed durable projection: {text}"); + assert!(report.lines >= 2, "{report:?}"); + assert_eq!(report.steps, 9, "each changed executable is counted once"); + assert_eq!(serde_json::to_value(&s.project).unwrap(), original, "live identifiers are restored exactly"); + let (clean, report) = crate::roles::script( + "# clean comment\r\nmap.zoomIn {}\r\nmap.zoomOut\nplugin:token-tools/read\nplugin:demo/token\ngeolibre:token\n", + 0, + ); + assert!(clean.is_none() && report.notices().is_empty(), "ordinary ids and formatting stay exact"); +} + +/// Cleaning both fields of one operation still affects one executable step. +#[test] +fn durable_executable_count_is_per_step() { + let mut project = Project::default(); + project.tasks.push(Task { + name: "One step".into(), + description: String::new(), + steps: vec![TaskStep { + title: String::new(), + instructions: String::new(), + command: format!("https://identifier.invalid/?token={SECRET}"), + params: json!({"password":SECRET}), + }], + }); + let (text, report) = project.saved_text(None).unwrap(); + assert!(!text.contains(SECRET)); + assert_eq!(report.steps, 1, "identifier and params belong to the same step: {report:?}"); +} + /// Inline synthetic state exercises owning fields without contacting any service. fn marked() -> Session { let mut s = world(); diff --git a/crates/engine/src/tests/service_logins.rs b/crates/engine/src/tests/service_logins.rs index 784d6b188..3d9657255 100644 --- a/crates/engine/src/tests/service_logins.rs +++ b/crates/engine/src/tests/service_logins.rs @@ -1279,16 +1279,21 @@ fn templates_tasks_and_models_written_out_keep_no_login_binding() { let tile = bound(&mut s, &id, "https://export757.example.org/tiles/{z}/{x}/{y}.png"); s.run("map.addTiles", &json!({"url": tile, "name": "Bound"})).unwrap(); s.run("task.new", &json!({"name":"Fetch","steps":[{"title":"Add","command":"map.addTiles","params":{"url": tile}}]})).unwrap(); - s.run("model.new", &json!({"name":"Tiles","steps":[{"tool":"buffer","params":{"input": tile, "distance": 1}}]})).unwrap(); - // A model drawn as a graph keeps its input's address in the node itself. - let graph = json!({"nodes":[{"id":"in","kind":"input","layer": tile}], "edges": []}); + // Keep the synthetic address in a tool's parameters, not a layer-name selector. + s.run("model.new", &json!({"name":"Tiles","graph":{"nodes":[{"id":"tool1","kind":"tool","tool":"buffer","params":{"input": tile,"distance":1}}],"edges":[]}})).unwrap(); + // A graph input selects an existing layer by name; its actual source fields are cleaned. + let graph = json!({"nodes":[{"id":"in","kind":"input","layer": "Cities"}], "edges": []}); s.run("model.new", &json!({"name":"Drawn","graph": graph})).unwrap(); let dir = std::env::temp_dir().join(format!("export757-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let (template, task) = (dir.join("bound.rgistemplate"), dir.join("bound.rgistask")); s.run("project.saveTemplate", &json!({"path": template.to_string_lossy()})).unwrap(); s.run("task.save", &json!({"task":"Fetch","path": task.to_string_lossy()})).unwrap(); - let model = s.run("model.export", &json!({"model":"Tiles"})).unwrap()["text"].as_str().unwrap().to_string(); + let model = s.run("model.export", &json!({"name":"Tiles"})).unwrap()["text"].as_str().unwrap().to_string(); + let drawn = s.run("model.export", &json!({"name":"Drawn"})).unwrap()["text"].as_str().unwrap().to_string(); + let graph: Value = serde_json::from_str(&drawn).unwrap(); + assert_eq!(graph["graph"]["nodes"][0]["layer"], "Cities"); + assert!(!drawn.contains(rustgis_data::login_address::PREFIX) && !drawn.contains("export-sentinel-757")); let written = [std::fs::read_to_string(&template).unwrap(), std::fs::read_to_string(&task).unwrap(), model]; let _ = std::fs::remove_dir_all(&dir); for text in &written { diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 80e3133b6..763f9b455 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -267,13 +267,12 @@ pub fn generation() -> u64 { let now = alarm_at(Instant::now()); let next = { let mut a = alarms().lock().unwrap_or_else(|e| e.into_inner()); - if a.first().is_some_and(|at| *at <= now) { + let (over, next) = take_due(&mut a, now); + if over { // Every wait over is one draw: the generation goes up once for them all. - let later = a.split_off(&now.saturating_add(1)); - *a = later; GENERATION.fetch_add(1, Ordering::Relaxed); } - a.first().copied() + next }; if let Some(at) = next { ring(Duration::from_millis(at.saturating_sub(now))); @@ -281,6 +280,16 @@ pub fn generation() -> u64 { GENERATION.load(Ordering::Relaxed) } +/// Consumes every elapsed deadline together and returns the next one to rearm. Keeping the +/// clock as an input lets the ordering contract be tested without scheduler deadlines. +fn take_due(alarms: &mut std::collections::BTreeSet, now: u64) -> (bool, Option) { + let over = alarms.first().is_some_and(|at| *at <= now); + if over { + *alarms = alarms.split_off(&now.saturating_add(1)); + } + (over, alarms.first().copied()) +} + /// How long until the earliest wait pending; none when there is none. #[cfg(test)] pub(crate) fn next_alarm() -> Option { @@ -2532,24 +2541,14 @@ mod tests { /// its elevation source waits), so the later is kept, and the app asked for it then. #[test] fn a_later_wait_isnt_lost_behind_an_earlier_one() { - let _alarm = super::alarm_turn(); - let asked = std::sync::Arc::new(Mutex::new(Vec::new())); - let noted = asked.clone(); - super::set_alarm(move |after| noted.lock().unwrap_or_else(|e| e.into_inner()).push(after)); - super::clear_alarms(); - let before = generation(); - notify_at(Instant::now() + Duration::from_millis(20)); - notify_at(Instant::now() + Duration::from_millis(150)); - std::thread::sleep(std::time::Duration::from_millis(50)); - let earlier = generation(); - assert!(earlier > before, "the earlier wait over"); - // The later is still pending, and asked of the app as the earlier was over. - assert!(super::next_alarm().is_some_and(|d| d <= Duration::from_millis(150)), "{:?}", super::next_alarm()); - let later = |asked: &Vec| asked.iter().any(|d| (Duration::from_millis(30)..=Duration::from_millis(150)).contains(d)); - assert!(later(&asked.lock().unwrap_or_else(|e| e.into_inner())), "{:?}", asked.lock().unwrap_or_else(|e| e.into_inner())); - std::thread::sleep(std::time::Duration::from_millis(150)); - assert!(generation() > earlier, "the later wait over"); - super::clear_alarm(); + let mut alarms = std::collections::BTreeSet::from([20, 150]); + assert_eq!(super::take_due(&mut alarms, 10), (false, Some(20))); + assert_eq!(super::take_due(&mut alarms, 50), (true, Some(150)), "rearm the later wait after the first expires"); + assert_eq!(super::take_due(&mut alarms, 100), (false, Some(150))); + assert_eq!(super::take_due(&mut alarms, 200), (true, None)); + assert_eq!(super::take_due(&mut alarms, 300), (false, None), "expired deadlines are consumed once"); + let mut delayed = std::collections::BTreeSet::from([20, 150]); + assert_eq!(super::take_due(&mut delayed, 200), (true, None), "a delayed poll correctly consumes both waits"); } /// A draw before a wait is over asks the app for it again: the app's request lasts one frame, diff --git a/crates/tiles/src/logins.rs b/crates/tiles/src/logins.rs index 420d3a26e..aaaaa9999 100644 --- a/crates/tiles/src/logins.rs +++ b/crates/tiles/src/logins.rs @@ -277,6 +277,9 @@ fn append_percent_encoded(result: &mut String, value: &str) { /// prevents overlapping values or a credential resembling `[redacted]` from leaking secrets. pub fn scrub(text: &str) -> String { let entries = entries(); + if entries.is_empty() && !text.contains(rustgis_data::login_address::PREFIX) { + return text.to_string(); + } let mut patterns: Vec<_> = entries.values().flat_map(|entry| &entry.redactions).collect(); patterns.sort_unstable_by_key(|pattern| std::cmp::Reverse(pattern.value.expose().len())); // Strip capabilities even after revocation, when there are no active credential patterns. From a00df9112b4ef21e3d5b4905a4c45428e65a6a1c Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 7 Nov 2026 03:18:47 -0600 Subject: [PATCH 4/4] fix(security): redact credentials from returned errors --- apps/rustgis-web/src/web.rs | 6 +-- crates/assistant/src/mcp.rs | 2 + crates/assistant/src/tests.rs | 12 +++++ crates/assistant/src/tools.rs | 84 +++++++++++++++++++++++++++++++++-- crates/tiles/src/cache.rs | 38 ++++++++++++---- crates/tiles/src/cog.rs | 2 +- crates/tiles/src/failure.rs | 6 ++- crates/tiles/src/logins.rs | 13 ++++++ crates/tiles/src/pmtiles.rs | 2 +- 9 files changed, 146 insertions(+), 19 deletions(-) diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index 0245bfb57..7d639e163 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -756,7 +756,7 @@ async fn fetch_bytes(url: &str, capture: Capture, omit: &'static [u16]) -> Resul } .await; logins::check(bound).map_err(str::to_string)?; - result.map_err(|error| if login.is_some() { logins::scrub(&format!("{url}: {error}")) } else { error }) + result.map_err(|error| if login.is_some() { logins::scrub_error(&format!("{url}: {error}")) } else { error }) } .await; diagnostics.record(answer.map_err(|e| Failure::new(url, e))).map_err(|failure| failure.message) @@ -820,7 +820,7 @@ async fn post_json(url: &str, body: &str, capture: Capture) -> Result, S } .await; logins::check(url).map_err(str::to_string)?; - result.map_err(|error| if login.is_some() { logins::scrub(&format!("{url}: {error}")) } else { error }) + result.map_err(|error| if login.is_some() { logins::scrub_error(&format!("{url}: {error}")) } else { error }) } .await; diagnostics.record(answer.map_err(|e| Failure::new(url, e))).map_err(|failure| failure.message) @@ -889,7 +889,7 @@ async fn head(url: &str, capture: Capture) -> Result bool { } } +/// Projects generated RPC error text while retaining the exact correlation id and code. fn error_value(id: &Value, code: i64, message: &str) -> Value { + let message = rustgis_engine::render::rustgis_tiles::logins::scrub_error(message); json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message}}) } diff --git a/crates/assistant/src/tests.rs b/crates/assistant/src/tests.rs index e04703a63..80cddbf73 100644 --- a/crates/assistant/src/tests.rs +++ b/crates/assistant/src/tests.rs @@ -1224,6 +1224,18 @@ fn mcp_protocol_errors() { assert_eq!(rpc(&server, &mut s, "tools/call", json!({"name": "no_such_tool"}))["error"]["code"], -32602); assert_eq!(rpc(&server, &mut s, "tools/call", json!({"name": "list_layers", "arguments": [1]}))["error"]["code"], -32602); assert_eq!(rpc(&server, &mut s, "resources/read", json!({"uri": "rustgis://docs/nope.md"}))["error"]["code"], -32002); + // Protocol validation fails before tool dispatch; its generated text is public too. + let private = "https://rpc-error.invalid/a?token=RGSYNTH_rpc_error&layer=roads"; + for (method, params, code) in [ + ("tools/call", json!({"name":private,"arguments":{}}), -32602), + ("resources/read", json!({"uri":private}), -32002), + (private, json!({}), -32601), + ] { + let reply = rpc(&server, &mut s, method, params); + assert_eq!(reply["error"]["code"], code); + assert!(!reply["error"]["message"].as_str().unwrap().contains("RGSYNTH_rpc_error")); + } + assert_eq!(rpc(&server, &mut s, "ping", json!({})), json!({})); // Batches: one reply per request, none for notifications only. let batch = r#"[{"jsonrpc":"2.0","id":1,"method":"ping"},{"jsonrpc":"2.0","method":"notifications/initialized"},{"jsonrpc":"2.0","id":2,"method":"nope"}]"#; let r: Value = serde_json::from_str(&server.handle(&mut s, batch).unwrap()).unwrap(); diff --git a/crates/assistant/src/tools.rs b/crates/assistant/src/tools.rs index 15a20386d..0dc2aa58a 100644 --- a/crates/assistant/src/tools.rs +++ b/crates/assistant/src/tools.rs @@ -259,6 +259,15 @@ pub fn run(s: &mut Session, call: &ToolCall) -> Result { /// Runs a call with a per-result token estimate, capped by the absolute ceiling. pub fn run_with_budget(s: &mut Session, call: &ToolCall, tokens: usize) -> Result { + // Script failures are structured JSON: clean their error field before serialization. + if call.name == "run" { + return run_script(s, &call.args, tokens); + } + run_with_budget_inner(s, call, tokens).map_err(|e| public_error(&e)) +} + +/// Assembles ordinary tool results; the public boundary cleans every early error too. +fn run_with_budget_inner(s: &mut Session, call: &ToolCall, tokens: usize) -> Result { let max_bytes = output_bytes(tokens); let a = &call.args; let v = match call.name.as_str() { @@ -278,7 +287,7 @@ pub fn run_with_budget(s: &mut Session, call: &ToolCall, tokens: usize) -> Resul return Err(format!("unknown command '{id}'; find ids with search_commands")); } let (before, mark) = (layer_snap(s), s.notice_mark()); - let r = s.run(&id, ¶ms_arg(a)?).map_err(|e| public_text(error_with_notices(e, s, mark)))?; + let r = s.run(&id, ¶ms_arg(a)?).map_err(|e| error_with_notices(e, s, mark))?; return Ok(command_result(s, r, &before, mark, tokens)); } "run_tool" => { @@ -286,12 +295,10 @@ pub fn run_with_budget(s: &mut Session, call: &ToolCall, tokens: usize) -> Resul let (before, mark) = (layer_snap(s), s.notice_mark()); let r = s .run("gp.run", &json!({"tool": tool, "params": params_arg(a)?})) - .map_err(|e| public_text(error_with_notices(e, s, mark)))?; + .map_err(|e| error_with_notices(e, s, mark))?; return Ok(command_result(s, r, &before, mark, tokens)); } "read_features" => read_features(s, a)?, - // MCP's `run` (not offered to the in-app assistant): a script in one call. - "run" => return run_script(s, a, tokens), "search_docs" => crate::docs::search(&str_arg(a, "query").unwrap_or_default(), limit_arg(a, 8, 30))?, "read_doc" => { return crate::docs::read_with_budget( @@ -331,6 +338,11 @@ fn public_text(text: String) -> String { rustgis_engine::render::rustgis_tiles::logins::scrub(&text) } +/// Generated errors may quote arbitrary addresses; successful values remain data. +fn public_error(text: &str) -> String { + rustgis_engine::render::rustgis_tiles::logins::scrub_error(text) +} + /// Publishes a completed result, reporting withheld subtrees before enforcing the response budget. #[cfg(test)] fn public_result(s: &mut Session, value: Value, tokens: usize) -> String { @@ -570,6 +582,7 @@ fn run_script(s: &mut Session, a: &Value, tokens: usize) -> Result { + let e = public_error(&e); // Nothing was kept, so no line ran and no layer was added. // The notice of the failure is the error redacted: compared as such, it isn't repeated. let told = rustgis_engine::data::diagnostics::redact(&e); @@ -921,6 +934,69 @@ pub fn context(s: &Session) -> String { #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { + /// Returned tool and fetch errors are cleaned before active patterns can damage URLs; + /// successful values and the original call remain outside the generated-error projection. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn generated_tool_and_fetch_errors_clean_addresses_before_active_patterns() { + const TEST: &str = "tools::tests::generated_tool_and_fetch_errors_clean_addresses_before_active_patterns"; + if std::env::var("RUSTGIS_ISOLATED_TOOL_ERRORS").as_deref() != Ok(TEST) { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST]) + .env("RUSTGIS_ISOLATED_TOOL_ERRORS", TEST) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("1 passed; 0 failed"), + "{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + return; + } + use rustgis_engine::render::rustgis_tiles::{failure::Failure, logins}; + let mut s = Session::new(rustgis_engine::Project::default()); + let url = "https://returned-error.invalid/a?token=RGSYNTH_returned_error&layer=roads"; + for active in [false, true] { + let binding = active.then(|| { + logins::activate( + "https://returned-error.invalid", + "synthetic-returned-error", + logins::Method::Header("X-Key".into()), + "https", + false, + ) + .unwrap() + }); + for (name, args) in [ + ("read_features", json!({"layer":url})), + ("describe_tool", json!({"tool":url})), + (url, json!({})), + ("run", json!({"script":url})), + ] { + let call = ToolCall { id: "error".into(), name: name.into(), args }; + let original = call.clone(); + let error = run_with_budget(&mut s, &call, DEFAULT_OUTPUT_TOKENS).unwrap_err(); + assert!(!error.contains("RGSYNTH_returned_error"), "{name}, active={active}: {error}"); + assert_eq!(call, original); + if name == "run" { + let reply: Value = serde_json::from_str(&error).unwrap(); + assert_eq!(reply["success"], false); + assert!(reply["error"].as_str().is_some()); + } + } + let address = binding.as_ref().map_or_else(|| url.to_string(), |binding| logins::bind(binding, url).unwrap()); + let failure = Failure::new(&address, format!("failed to fetch {address}")); + assert!(!failure.message.contains("RGSYNTH_returned_error")); + if let Some(binding) = binding { + assert!(logins::deactivate(&binding)); + } + } + let clean = ToolCall { id: "clean".into(), name: "read_features".into(), args: json!({"layer":"ordinary missing layer"}) }; + assert!(run_with_budget(&mut s, &clean, DEFAULT_OUTPUT_TOKENS).unwrap_err().contains("ordinary missing layer")); + assert_eq!(Failure::new("https://clean.invalid/a?layer=roads", "HTTP 403").message, "HTTP 403"); + assert_eq!(logins::scrub_error("failed https://clean.invalid/a?layer=roads"), "failed https://clean.invalid/a?layer=roads"); + } use super::*; use rustgis_engine::render::rustgis_tiles::logins; diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 763f9b455..1e404fbbf 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -145,7 +145,8 @@ pub fn clear_fetcher() { /// on whichever thread is starting them already, so one test's fetch may run on another's thread, /// later. Tests that end a login take turns too, as that empties the cache ([`clear`]), and so do /// tests that wait for or plant what the cache holds (an archive read, a style resolved), which -/// that would drop. The turn puts the native fetcher back when it ends. +/// that would drop. Tests that inspect clean failure text also take turns: a concurrent login +/// can redact even a single letter from that text. The turn puts the native fetcher back when it ends. #[cfg(test)] pub(crate) fn turn() -> Turn { static TURN: Mutex<()> = Mutex::new(()); @@ -692,7 +693,7 @@ fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16]) { Entry::Ready(Arc::new(p), size) } Err(e) => { - log::debug!("{}", crate::logins::scrub(&format!("{}: {e}", answer.key))); + log::debug!("{}", crate::logins::scrub_error(&format!("{}: {e}", answer.key))); // Refused until a time its server gave: the map draws again then. if let Some(wait) = retry_after(&e) { notify_at(Instant::now() + wait); @@ -876,7 +877,7 @@ impl<'a> RequestDiagnostics<'a> { let url = crate::logins::inner(url); let url = if rustgis_data::login_address::is_wrapper(url) { "[address omitted]" } else { url }; let clean = |text: &str| { - if carried || text.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub(text) } else { text.to_string() } + if carried || text.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub_error(text) } else { text.to_string() } }; // A status the caller expects is an answer, not a failure for the history; a caller // that treats the refusal as an error says so itself. One bounded record per server says @@ -1626,7 +1627,9 @@ mod native { } Err("too many redirects".into()) })(); - diagnostics.record(answer).map_err(|failure| if carried_login { crate::logins::scrub(&failure.message) } else { failure.message }) + diagnostics + .record(answer) + .map_err(|failure| if carried_login { crate::logins::scrub_error(&failure.message) } else { failure.message }) } /// POSTs a JSON document on a thread of its own. @@ -1682,7 +1685,9 @@ mod native { crate::logins::check(url).map_err(str::to_string)?; Ok(buf) })(); - diagnostics.record(answer).map_err(|failure| if carried_login { crate::logins::scrub(&failure.message) } else { failure.message }) + diagnostics + .record(answer) + .map_err(|failure| if carried_login { crate::logins::scrub_error(&failure.message) } else { failure.message }) } /// Downloads on a thread of its own, so a large file doesn't hold up the tiles. @@ -1785,7 +1790,7 @@ mod native { /// [`get_detailed`], a status in `omit` not recorded as a failure. pub(super) fn get_detailed_omitting(agent: &ureq::Agent, url: &str, omit: &'static [u16]) -> Result, Failure> { fetch_answer(agent, url, true, omit).map_err(|f| Failure { - url: if url.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub(url) } else { url.to_string() }, + url: if url.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub_error(url) } else { url.to_string() }, ..f }) } @@ -2004,8 +2009,8 @@ mod native { let result = crate::logins::check(super::resource_of(url)).map_err(Failure::from).and(result); result.map_err(|mut f| { if carried_login { - f.message = crate::logins::scrub(&f.message); - f.url = crate::logins::scrub(&f.url); + f.message = crate::logins::scrub_error(&f.message); + f.url = crate::logins::scrub_error(&f.url); } f }) @@ -2052,6 +2057,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_capture_background_failures_once() { + let _turn = turn(); use rustgis_data::diagnostics::{History, NetworkKind}; let history = History::default(); let _scope = history.enter(); @@ -2072,6 +2078,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_interleaved_sessions_keep_their_requests_private() { + let _turn = turn(); use rustgis_data::diagnostics::History; use std::io::{BufRead, BufReader, Write}; let first = History::default(); @@ -2132,6 +2139,7 @@ mod tests { #[test] fn network_diagnostics_explicit_capture_outlives_dispatch_scope_not_owner() { + let _turn = turn(); use rustgis_data::diagnostics::History; let first = History::default(); let second = History::default(); @@ -2158,6 +2166,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_redact_downloads_without_retaining_bodies() { + let _turn = turn(); use rustgis_data::diagnostics::History; let history = History::default(); let _scope = history.enter(); @@ -2208,6 +2217,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_skip_gone_map_tiles_and_keep_server_errors() { + let _turn = turn(); let history = rustgis_data::diagnostics::History::default(); let _scope = history.enter(); let answer = |status: &str| format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); @@ -2229,6 +2239,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_keep_headers_after_a_credential_header() { + let _turn = turn(); let history = rustgis_data::diagnostics::History::default(); let _scope = history.enter(); let base = serve(vec![ @@ -2251,6 +2262,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_skip_routine_tile_404s_and_keep_other_statuses() { + let _turn = turn(); use rustgis_data::diagnostics::{History, Severity}; let history = History::default(); let _scope = history.enter(); @@ -2309,6 +2321,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_keep_observed_status_on_blocking_body_failure() { + let _turn = turn(); use rustgis_data::diagnostics::{History, NetworkKind}; let history = History::default(); let _scope = history.enter(); @@ -2326,6 +2339,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_preserve_connection_and_timeout_causes() { + let _turn = turn(); use rustgis_data::diagnostics::{History, NetworkKind}; let history = History::default(); let _scope = history.enter(); @@ -2360,6 +2374,7 @@ mod tests { /// history keeps no record of it. #[test] fn network_diagnostics_skip_failed_local_reads() { + let _turn = turn(); let history = rustgis_data::diagnostics::History::default(); let _scope = history.enter(); native::get(&native::agent(), "local-file:/nonexistent-1155.pmtiles", &[]).unwrap_err(); @@ -2368,6 +2383,7 @@ mod tests { #[test] fn network_diagnostics_ignore_cancellation_and_keep_browser_causes_truthful() { + let _turn = turn(); use rustgis_data::diagnostics::{History, NetworkKind}; let history = History::default(); let _scope = history.enter(); @@ -2395,6 +2411,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn public_downloads_refuse_names_that_resolve_locally() { + let _turn = turn(); let err = download_public_within("http://localhost:9/doc.json", 1024, std::time::Duration::from_secs(5)).unwrap_err(); assert!(err.contains("resolves to this computer or the local network"), "{err}"); } @@ -2422,6 +2439,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn downloads_stop_at_the_callers_budget() { + let _turn = turn(); for framing in ["Content-Length: 4\r\n", "", "Transfer-Encoding: chunked\r\n"] { let body = if framing.starts_with("Transfer") { "4\r\ntile\r\n0\r\n\r\n" } else { "tile" }; for (limit, ok) in [(0, false), (3, false), (4, true), (5, true)] { @@ -2442,6 +2460,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn download_budget_also_bounds_decompressed_bytes() { + let _turn = turn(); let data = vec![b'x'; 4096]; // The ZIP writer supplies the CRC-32 for a valid, generated gzip trailer. let zip = rustgis_data::zip::write(&[("body".into(), data.clone())]).unwrap(); @@ -3080,6 +3099,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn the_length_asked_for_follows_redirects_and_keeps_refusals() { + let _turn = turn(); let no_suffix = "HTTP/1.1 501 Not Implemented\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(); let agent = native::agent(); let base = serve(vec![ @@ -3114,6 +3134,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn a_refused_download_says_who_refused_it() { + let _turn = turn(); let page = "Attention Required! | CloudflareSorry, you have been blocked"; let blocked = format!( "HTTP/1.1 403 Forbidden\r\nServer: cloudflare\r\nCF-RAY: a4c9-BOG\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{page}", @@ -3135,6 +3156,7 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] #[test] fn follows_only_allowed_redirects() { + let _turn = turn(); let redirect = |to: &str| format!("HTTP/1.1 302 Found\r\nLocation: {to}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); let ok = "HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntile".to_string(); let agent = native::agent(); diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index f1dc6fe9a..09c867cf9 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -404,7 +404,7 @@ fn with_info(url: &str, then: Box, String>) + Send>) let r = crate::logins::check(&key) .map_err(str::to_string) .and(r) - .map_err(|e| if rustgis_data::login_address::is_wrapper(&key) { crate::logins::scrub(&e) } else { e }); + .map_err(|e| if rustgis_data::login_address::is_wrapper(&key) { crate::logins::scrub_error(&e) } else { e }); let waiting = { let mut i = infos().lock().unwrap_or_else(|e| e.into_inner()); let next = match &r { diff --git a/crates/tiles/src/failure.rs b/crates/tiles/src/failure.rs index 0f5c94eef..58cbf0b3a 100644 --- a/crates/tiles/src/failure.rs +++ b/crates/tiles/src/failure.rs @@ -68,9 +68,9 @@ impl Failure { /// the browser's `HTTP 403`). pub fn new(url: &str, message: impl Into) -> Self { let message = message.into(); - let message = if message.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub(&message) } else { message }; let reason = crate::cache::http_status(&message).filter(|s| *s >= 400).map_or(Reason::Other, Reason::Status); - let url = if url.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub(url) } else { url.to_string() }; + let message = crate::logins::scrub_error(&message); + let url = if url.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub_error(url) } else { url.to_string() }; Failure { url, message, reason, answer: None } } @@ -375,6 +375,7 @@ mod tests { #[test] fn a_cloudflare_block_is_named_with_its_page_title() { + let _turn = crate::cache::turn(); let body = b"Attention Required! | CloudflareSorry, you have been blocked"; let f = Failure::answered( "https://example.org/MapServer?f=json", @@ -462,6 +463,7 @@ mod tests { /// read, say nothing more. #[test] fn retry_after_on_429_and_503() { + let _turn = crate::cache::turn(); let base = || "http status: 503".to_string(); assert_eq!(with_retry_after(base(), 503, Some("30")), "http status: 503 (retry after 30 s)"); assert_eq!(with_retry_after("HTTP 429".into(), 429, Some(" 2 ")), "HTTP 429 (retry after 2 s)"); diff --git a/crates/tiles/src/logins.rs b/crates/tiles/src/logins.rs index aaaaa9999..7c92a3498 100644 --- a/crates/tiles/src/logins.rs +++ b/crates/tiles/src/logins.rs @@ -282,6 +282,19 @@ pub fn scrub(text: &str) -> String { } let mut patterns: Vec<_> = entries.values().flat_map(|entry| &entry.redactions).collect(); patterns.sort_unstable_by_key(|pattern| std::cmp::Reverse(pattern.value.expose().len())); + scrub_patterns(text, &patterns) +} + +/// Cleans generated error text before active values can obscure its address syntax. +/// Runtime request bindings and successful data values must not use this output projection. +pub fn scrub_error(text: &str) -> String { + let unbound = scrub_patterns(text, &[]); + let (public, _) = rustgis_data::address::scrub_addresses(&unbound); + scrub(&public) +} + +/// Removes runtime capability headers and the supplied active patterns in one pass. +fn scrub_patterns(text: &str, patterns: &[&Redaction]) -> String { // Strip capabilities even after revocation, when there are no active credential patterns. let mut result = String::with_capacity(text.len()); let mut rest = text; diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index 1176b8548..8a5cb2dc5 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -512,7 +512,7 @@ fn finish(archive: &str, r: Result) { .map_err(str::to_string) .and(r) .map(Arc::new) - .map_err(|e| if rustgis_data::login_address::is_wrapper(archive) { crate::logins::scrub(&e) } else { e }); + .map_err(|e| if rustgis_data::login_address::is_wrapper(archive) { crate::logins::scrub_error(&e) } else { e }); let waiting = { let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); let old = match &r {