Upgrade pod2 for sparse array literals

Cargo.toml

@@ -22,7 +22,7 @@ members = [
 resolver = "2"
 
 [workspace.dependencies]
-pod2 = { git = "https://github.com/0xPARC/pod2", rev = "1ee11948bcb83f53b689f0a9a14a68b943120c2e", default-features = false, features = [
+pod2 = { git = "https://github.com/0xPARC/pod2", rev = "a91d87895de0f6d9bda256016572d593216e619a", default-features = false, features = [
     "backend_plonky2",
     "disk_cache",
     "zk",

libs/sdk/README.md

@@ -169,9 +169,7 @@ representation in this order: set, dictionary, array. Because statement
 arguments are compared and hashed by raw value, this choice does not affect
 verification.
 
-The pinned pod2 version requires contiguous indexes in Podlang array literals.
-Sparse arrays are rendered as commitments with a warning. Support for rendering
-them as literals can follow a dependency update that includes
+Sparse arrays are rendered with the indexed Podlang literal syntax added in
 [pod2 #541](https://github.com/0xPARC/pod2/pull/541).
 
 ## Type checking

libs/sdk/src/fmt_podlang.rs

@@ -192,9 +192,8 @@ struct ArgFmt<'a> {
 /// prefer set, then dictionary, then array. Each representation has the same
 /// raw value, which is what statement arguments use.
 ///
-/// The pinned pod2 version cannot represent sparse arrays in Podlang, so they
-/// are rendered as their commitment. Once the dependency includes
-/// <https://github.com/0xPARC/pod2/pull/541>, render them with sparse-array syntax.
+/// Sparse arrays use the indexed literal syntax added in
+/// <https://github.com/0xPARC/pod2/pull/541>.
 struct LiteralFmt<'a>(&'a Value);
 
 #[cfg(test)]
@@ -240,7 +239,6 @@ impl<'a> fmt::Display for LiteralFmt<'a> {
                 .collect::<Result<Vec<_>, _>>()
                 .map_err(|_| fmt::Error)?;
             slots.sort_by_key(|(index, _)| *index);
-            // Podlang array literals cannot represent gaps between indexes.
             if slots.iter().enumerate().all(|(i, (index, _))| i == *index) {
                 write!(f, "[")?;
                 for (i, (_, element)) in slots.iter().enumerate() {
@@ -249,11 +247,12 @@ impl<'a> fmt::Display for LiteralFmt<'a> {
                 }
                 return write!(f, "]");
             } else {
-                log::warn!(
-                    "Rendering sparse array as its commitment: the pinned pod2 version \
-                     does not support sparse-array literals"
-                );
-                return write!(f, "{}", Value::from(value.raw()));
+                write!(f, "[")?;
+                for (i, (index, element)) in slots.iter().enumerate() {
+                    let sep = if i == 0 { "" } else { ", " };
+                    write!(f, "{sep}{index}: {}", LiteralFmt(element))?;
+                }
+                return write!(f, "]");
             }
         }
         write!(f, "{}", Value::from(value.raw()))

libs/sdk/src/lib.rs

@@ -11,7 +11,7 @@ use itertools::zip_eq;
 use lt_eq_u256_pod::{LtEqU256Pod, STANDARD_LT_EQ_U256_VD_HASH};
 use pod2::{
     backends::plonky2::{basetypes::DEFAULT_VD_SET, mainpod::Prover, mock::mainpod::MockProver},
-    frontend::{MainPod, MultiPodBuilder, Operation, OperationArg},
+    frontend::{MainPod, MultiPodBuilder, Operation, OperationArg, entry},
     lang::{Module, load_module},
     middleware::{
         EMPTY_VALUE, F, Hash, MainPodProver, NativePredicate, OperationAux, OperationType, Params,
@@ -968,23 +968,25 @@ impl ActionHandle {
         // Resolve an Output's pre-identity dict to its slot in the
         // `<Action>Initials` record.
         let initials_anchor = |obj_name: &str| -> Option<OperationArg> {
-            let (slot, entry) = meta.initials_entry(obj_name)?;
-            if entry.needs_wildcard {
+            let (slot, shape) = meta.initials_entry(obj_name)?;
+            if shape.needs_wildcard {
                 return None;
             }
-            Some((initials_array.as_ref()?, slot as i64).into())
+            Some(entry(initials_array.as_ref()?, slot as i64).expect("initials slot exists"))
         };
 
         // Returns the anchored op-arg for the Object's collapsed side at
         // this ts, or None when the form stays an explicit wildcard.
         let anchor_at = |obj_name: &str, ts: usize| -> Option<OperationArg> {
             match meta.collapsed_at(obj_name, ts)? {
-                fmt_podlang::Collapse::IO(fmt_podlang::Side::In) => {
-                    Some((&io_array, meta.in_entry(obj_name).unwrap().0 as i64).into())
-                }
-                fmt_podlang::Collapse::IO(fmt_podlang::Side::Out) => {
-                    Some((&io_array, meta.out_entry(obj_name).unwrap().0 as i64).into())
-                }
+                fmt_podlang::Collapse::IO(fmt_podlang::Side::In) => Some(
+                    entry(&io_array, meta.in_entry(obj_name).unwrap().0 as i64)
+                        .expect("input slot exists"),
+                ),
+                fmt_podlang::Collapse::IO(fmt_podlang::Side::Out) => Some(
+                    entry(&io_array, meta.out_entry(obj_name).unwrap().0 as i64)
+                        .expect("output slot exists"),
+                ),
                 fmt_podlang::Collapse::Initials => {
                     Some(initials_anchor(obj_name).expect("collapsed_at promised an initials slot"))
                 }
@@ -1229,7 +1231,7 @@ impl ActionHandle {
         // for actions with no intermediates.
         let chain_step_anchor = |ts: usize| -> Option<OperationArg> {
             let slot = fmt_podlang::chain_step_at(ts, chain_max_ts)?;
-            Some((chain_steps_array.as_ref()?, slot as i64).into())
+            Some(entry(chain_steps_array.as_ref()?, slot as i64).expect("chain step slot exists"))
         };
 
         // Each pending Tx event is first proved with literal args. We then use
@@ -1247,14 +1249,14 @@ impl ActionHandle {
                     ObjectIO::Input | ObjectIO::Mutate => meta
                         .in_entry(varname)
                         .filter(|(_, e)| !e.needs_wildcard)
-                        .map(|(idx, _)| (&io_array, idx as i64).into()),
+                        .map(|(idx, _)| entry(&io_array, idx as i64).expect("input slot exists")),
                     ObjectIO::Output => None,
                 };
                 let new_anchor: Option<OperationArg> = match io {
                     ObjectIO::Output | ObjectIO::Mutate => meta
                         .out_entry(varname)
                         .filter(|(_, e)| !e.needs_wildcard)
-                        .map(|(idx, _)| (&io_array, idx as i64).into()),
+                        .map(|(idx, _)| entry(&io_array, idx as i64).expect("output slot exists")),
                     ObjectIO::Input => None,
                 };
                 let pre_ts = pending.post_ts - 1;

libs/sdk/src/tests.rs

@@ -1517,11 +1517,10 @@ fn test_literal_container_rejects_var() {
     assert!(err.contains("is a var, not a literal"), "{err}");
 }
 
-/// Renders sparse arrays as commitments because the pinned pod2 version has no
-/// sparse-array literal syntax. Update this expectation once the dependency
-/// includes <https://github.com/0xPARC/pod2/pull/541>.
+/// Renders sparse arrays with the indexed syntax added in
+/// <https://github.com/0xPARC/pod2/pull/541>.
 #[test]
-fn test_sparse_array_literal_renders_as_commitment() {
+fn test_sparse_array_literal_renders_with_indexes() {
     let dense = Array::new(vec![Value::from(1), Value::from(2)]);
     assert_eq!(
         fmt_podlang::literal_podlang(&Value::from(dense)),
@@ -1530,11 +1529,21 @@ fn test_sparse_array_literal_renders_as_commitment() {
 
     let mut sparse = Array::empty_with_db(Box::new(pod2::middleware::db::mem::MemDB::new()));
     sparse.insert(5, Value::from(1)).unwrap();
+    sparse.insert(7, Value::from(2)).unwrap();
     let sparse = Value::from(sparse);
-    assert_eq!(
-        fmt_podlang::literal_podlang(&sparse),
-        Value::from(sparse.raw()).to_string()
-    );
+    let literal = fmt_podlang::literal_podlang(&sparse);
+    assert_eq!(literal, "[5: 1, 7: 2]");
+
+    // The rendered source must lower back to the same container, rather than
+    // merely pass the Podlang parser.
+    let source = format!("my_pred(A) = AND(Equal(A, {literal}))");
+    let module = pod2::lang::load_module(&source, "sparse_literal", &Params::default(), &[])
+        .expect("indexed array literal lowers");
+    let arg = &module.batch.predicates()[0].statements()[0].args()[1];
+    let pod2::middleware::StatementTmplArg::Literal(lowered) = arg else {
+        panic!("expected a literal, got {arg:?}");
+    };
+    assert_eq!(lowered.raw(), sparse.raw());
 }
 
 /// Reports an execution error when a variable-key lookup misses.

libs/txlib/src/lib.rs

@@ -32,7 +32,7 @@ use std::{collections::HashMap, sync::Arc};
 
 use pod2::{
     backends::plonky2::primitives::merkletree::MerkleProof,
-    frontend::Operation,
+    frontend::{Operation, entry},
     middleware::{
         EMPTY_VALUE, Hash, NativeOperation, OperationAux, OperationType, Statement, StrKey, Value,
         containers::{Array, Dictionary, Set},
@@ -788,8 +788,8 @@ impl TxBuilder {
         let st_eq_stable_identifier = ctx
             .builder
             .priv_op(op!(Equal(
-                (old, STABLE_IDENTIFIER_FIELD),
-                (new, STABLE_IDENTIFIER_FIELD)
+                entry(old, STABLE_IDENTIFIER_FIELD).expect("referenced container entry exists"),
+                entry(new, STABLE_IDENTIFIER_FIELD).expect("referenced container entry exists")
             )))
             .unwrap();
         let st_h1 = ctx
@@ -910,7 +910,10 @@ impl TxBuilder {
         let st_inputs_rebound = ctx
             .builder
             .priv_op(Operation::replace_value_with_entry(
-                vec![Some((&before_tx, "live")), None],
+                vec![
+                    Some(entry(&before_tx, "live").expect("referenced container entry exists")),
+                    None,
+                ],
                 self.st_inputs_grounded.clone(),
             ))
             .unwrap();
@@ -920,7 +923,10 @@ impl TxBuilder {
             .priv_op(Operation::replace_value_with_entry(
                 vec![
                     None,
-                    Some((&state_header_arr, STATE_HEADER_CREATED_SLOT as i64)),
+                    Some(
+                        entry(&state_header_arr, STATE_HEADER_CREATED_SLOT as i64)
+                            .expect("referenced container entry exists"),
+                    ),
                 ],
                 st_inputs_rebound,
             ))
@@ -932,7 +938,11 @@ impl TxBuilder {
         let st_hash_rebound = ctx
             .builder
             .priv_op(Operation::replace_value_with_entry(
-                vec![Some((&before_tx, "live")), None, None],
+                vec![
+                    Some(entry(&before_tx, "live").expect("referenced container entry exists")),
+                    None,
+                    None,
+                ],
                 st_hash,
             ))
             .unwrap();
@@ -958,7 +968,12 @@ impl TxBuilder {
         let st_dict_insert = ctx
             .builder
             .priv_op(Operation::replace_value_with_entry(
-                vec![None, None, Some((&before_tx, "live")), None],
+                vec![
+                    None,
+                    None,
+                    Some(entry(&before_tx, "live").expect("referenced container entry exists")),
+                    None,
+                ],
                 st_dict_insert_lit,
             ))
             .unwrap();
@@ -1421,11 +1436,18 @@ mod tests {
             let (st_mutate, h_sub) = tx2.mutate(&mut ctx, &pick_new, &pick);
             let op_gt = ctx
                 .builder
-                .priv_op(op!(Gt((&pick, "durability"), 0_i64)))
+                .priv_op(op!(Gt(
+                    entry(&pick, "durability").expect("referenced container entry exists"),
+                    0_i64
+                )))
                 .unwrap();
             let op_sum = ctx
                 .builder
-                .priv_op(op!(Sum(99_i64, 1_i64, (&pick, "durability"))))
+                .priv_op(op!(Sum(
+                    99_i64,
+                    1_i64,
+                    entry(&pick, "durability").expect("referenced container entry exists")
+                )))
                 .unwrap();
             let op_du = ctx
                 .builder
@@ -1650,14 +1672,26 @@ mod tests {
         let st_ins_a_anchored = ctx
             .builder
             .priv_op(Operation::replace_value_with_entry(
-                vec![None, None, Some((&initials, "stick_a")), None, None],
+                vec![
+                    None,
+                    None,
+                    Some(entry(&initials, "stick_a").expect("referenced container entry exists")),
+                    None,
+                    None,
+                ],
                 st_ins_a,
             ))
             .unwrap();
         let st_ins_b_anchored = ctx
             .builder
             .priv_op(Operation::replace_value_with_entry(
-                vec![None, None, Some((&initials, "stick_b")), None, None],
+                vec![
+                    None,
+                    None,
+                    Some(entry(&initials, "stick_b").expect("referenced container entry exists")),
+                    None,
+                    None,
+                ],
                 st_ins_b,
             ))
             .unwrap();