From 0223edc4d7000742af645005c9b5728449213505 Mon Sep 17 00:00:00 2001 From: JetF0x <42677581+JetF0x@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:15:33 -0500 Subject: [PATCH] Add raw field copying for schema upgrades --- .../Entities/UpgradeCopyTestTypes.cs | 93 ++++++ .../SerializerRuntimeTests.UpgradeCopy.cs | 270 ++++++++++++++++++ Cat.Network/NetworkObjectUpgradeReader.cs | 15 +- Cat.Network/NetworkObjectUpgradeWriter.cs | 23 ++ README.md | 20 +- 5 files changed, 414 insertions(+), 7 deletions(-) create mode 100644 Cat.Network.Test/Entities/UpgradeCopyTestTypes.cs create mode 100644 Cat.Network.Test/SerializerRuntimeTests.UpgradeCopy.cs diff --git a/Cat.Network.Test/Entities/UpgradeCopyTestTypes.cs b/Cat.Network.Test/Entities/UpgradeCopyTestTypes.cs new file mode 100644 index 0000000..d63d4ca --- /dev/null +++ b/Cat.Network.Test/Entities/UpgradeCopyTestTypes.cs @@ -0,0 +1,93 @@ +namespace Cat.Network.Test.Entities; + +[NetworkObject(Version = 1)] +public partial class LegacyUpgradeCollectionsState : NetworkObject { + [NetworkProperty] + public partial int Marker { get; set; } + + [NetworkProperty] + public partial int? OptionalValue { get; set; } + + [NetworkProperty] + public partial DirtyChildState? Child { get; set; } + + [NetworkCollection] + public partial NetworkList Values { get; } + + [NetworkCollection] + public partial NetworkDictionary Labels { get; } + + [NetworkCollection] + public partial NetworkList NullableValues { get; } + + [NetworkCollection] + public partial NetworkList Children { get; } + + [NetworkCollection] + public partial NetworkDictionary ChildLookup { get; } +} + +[NetworkObject(Version = 2)] +public partial class RenamedUpgradeCollectionsState : NetworkObject { + [NetworkProperty] + public partial int Marker { get; set; } + + [NetworkProperty] + public partial int? OptionalCount { get; set; } + + [NetworkProperty] + public partial DirtyChildState? Detail { get; set; } + + [NetworkCollection] + public partial NetworkList Items { get; } + + [NetworkCollection] + public partial NetworkDictionary Names { get; } + + [NetworkCollection] + public partial NetworkList OptionalItems { get; } + + [NetworkCollection] + public partial NetworkList NestedItems { get; } + + [NetworkCollection] + public partial NetworkDictionary NestedLookup { get; } + + [UpgradeTo(2)] + internal static void UpgradeToVersion2(NetworkObjectUpgradeReader reader, NetworkObjectUpgradeWriter writer) { + writer.CopyExcept("Values", "Labels", "OptionalValue", "Child", "NullableValues", "Children", "ChildLookup"); + writer.Copy("Values", "Items"); + writer.Copy("Labels", "Names"); + writer.Copy("OptionalValue", "OptionalCount"); + writer.Copy("Child", "Detail"); + writer.Copy("NullableValues", "OptionalItems"); + writer.Copy("Children", "NestedItems"); + writer.Copy("ChildLookup", "NestedLookup"); + } +} + +[NetworkObject(Version = 2)] +public partial class UnchangedUpgradeCollectionsState : NetworkObject { + [NetworkProperty] + public partial int Marker { get; set; } + + [NetworkCollection] + public partial NetworkList Values { get; } + + [NetworkCollection] + public partial NetworkDictionary Labels { get; } + + [UpgradeTo(2)] + internal static void UpgradeToVersion2(NetworkObjectUpgradeReader reader, NetworkObjectUpgradeWriter writer) { + writer.CopyExcept(); + } +} + +[NetworkObject] +public partial class UpgradeNestedCollectionState : NetworkObject { + [NetworkCollection] + public partial NetworkList Values { get; } + + [NetworkCollection] + public partial NetworkDictionary Labels { get; } +} diff --git a/Cat.Network.Test/SerializerRuntimeTests.UpgradeCopy.cs b/Cat.Network.Test/SerializerRuntimeTests.UpgradeCopy.cs new file mode 100644 index 0000000..758c3df --- /dev/null +++ b/Cat.Network.Test/SerializerRuntimeTests.UpgradeCopy.cs @@ -0,0 +1,270 @@ +using System.Buffers.Binary; +using Cat.Network.Test.Entities; + +namespace Cat.Network.Test; + +public sealed partial class SerializerRuntimeTests { + [TestCase(false)] + [TestCase(true)] + public void UpgradeCopy_GeneratedMigrationRenamesListsAndDictionaries(bool empty) { + TypeCatalogue catalogue = RegisterUpgradeCopyTypes(); + LegacyUpgradeCollectionsState source = new() { Marker = 42 }; + if (!empty) { + source.Values.Add(3); + source.Values.Add(5); + source.Labels.Add(7, "seven"); + source.Labels.Add(8, ""); + source.Labels.Add(9, "Mira 猫"); + } + byte[] payload = Serialize(source, catalogue, MemberIdentificationMode.Name); + RenamedUpgradeCollectionsState target = new(); + target.Items.Add(-1); + target.Names.Add(-1, "replaced"); + + Deserialize(target, catalogue, payload); + + Assert.Multiple(() => { + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(payload), Is.EqualTo(1)); + Assert.That(target.Marker, Is.EqualTo(42)); + Assert.That(target.Items, Is.EqualTo(source.Values)); + Assert.That(target.Names, Is.EquivalentTo(source.Labels)); + }); + AssertRenamedUpgradeBytesArePreserved(payload, catalogue); + } + + [TestCase(false)] + [TestCase(true)] + public void UpgradeCopy_GeneratedMigrationPreservesNullableObjectsAndNestedCollections(bool nullProperties) { + TypeCatalogue catalogue = RegisterUpgradeCopyTypes(); + LegacyUpgradeCollectionsState source = new() { + OptionalValue = nullProperties ? null : 17, + Child = nullProperties ? null : new DirtyChildState { Value = 23 } + }; + source.NullableValues.Add(null); + source.NullableValues.Add(31); + source.Children.Add(null); + source.Children.Add(CreateUpgradeNestedCollection(37)); + source.ChildLookup.Add(1, null); + source.ChildLookup.Add(2, CreateUpgradeNestedCollection(41)); + byte[] payload = Serialize(source, catalogue, MemberIdentificationMode.Name); + RenamedUpgradeCollectionsState target = new() { + OptionalCount = -1, + Detail = new DirtyChildState { Value = -1 } + }; + + Deserialize(target, catalogue, payload); + + Assert.Multiple(() => { + Assert.That(target.OptionalCount, Is.EqualTo(source.OptionalValue)); + Assert.That(target.Detail?.Value, Is.EqualTo(source.Child?.Value)); + Assert.That(target.OptionalItems, Is.EqualTo(new int?[] { null, 31 })); + Assert.That(target.NestedItems, Has.Count.EqualTo(2)); + Assert.That(target.NestedItems[0], Is.Null); + Assert.That(target.NestedItems[1]!.Values, Is.EqualTo(new int?[] { null, 37 })); + Assert.That(target.NestedItems[1]!.Labels[37], Is.EqualTo("nested 猫")); + Assert.That(target.NestedLookup, Has.Count.EqualTo(2)); + Assert.That(target.NestedLookup[1], Is.Null); + Assert.That(target.NestedLookup[2]!.Values, Is.EqualTo(new int?[] { null, 41 })); + Assert.That(target.NestedLookup[2]!.Labels[41], Is.EqualTo("nested 猫")); + }); + AssertRenamedUpgradeBytesArePreserved(payload, catalogue); + } + + [Test] + public void UpgradeCopy_CopyExceptStillPreservesUnchangedGeneratedCollectionPayloads() { + TypeCatalogue catalogue = RegisterUpgradeCopyTypes(); + LegacyUpgradeCollectionsState source = new() { Marker = 42 }; + source.Values.Add(3); + source.Values.Add(5); + source.Labels.Add(7, "seven"); + byte[] payload = Serialize(source, catalogue, MemberIdentificationMode.Name); + SerializationContext context = new(catalogue); + NetworkObjectUpgradeReader reader = ReadUpgradeFields(payload.AsSpan(2), context); + NetworkObjectUpgradeWriter writer = new(new BufferWriter(), reader, context); + + UnchangedUpgradeCollectionsState.UpgradeToVersion2(reader, writer); + UnchangedUpgradeCollectionsState target = new(); + Deserialize(target, catalogue, payload); + + Assert.Multiple(() => { + Assert.That(writer.Complete(), Is.EqualTo(payload[2..])); + Assert.That(target.Marker, Is.EqualTo(42)); + Assert.That(target.Values, Is.EqualTo(new[] { 3, 5 })); + Assert.That(target.Labels, Is.EquivalentTo(source.Labels)); + }); + } + + [Test] + public void UpgradeCopy_TypedCollectionReadsAndWritesRemainUnsupported() { + TypeCatalogue catalogue = RegisterUpgradeCopyTypes(); + LegacyUpgradeCollectionsState source = new(); + source.Values.Add(3); + source.Labels.Add(7, "seven"); + SerializationContext context = new(catalogue); + NetworkObjectUpgradeReader reader = ReadUpgradeFields(Serialize(source, catalogue, MemberIdentificationMode.Name).AsSpan(2), context); + NetworkObjectUpgradeWriter writer = new(new BufferWriter(), reader, context); + + Assert.Multiple(() => { + Assert.That(() => reader.Get>("Values"), Throws.InvalidOperationException.With.Message.Contains("is not supported")); + Assert.That(() => reader.Get>("Labels"), Throws.InvalidOperationException.With.Message.Contains("is not supported")); + Assert.That(() => writer.Write("Items", source.Values), Throws.InvalidOperationException.With.Message.Contains("is not supported")); + Assert.That(() => writer.Write("Names", source.Labels), Throws.InvalidOperationException.With.Message.Contains("is not supported")); + Assert.That(ReadUpgradeFields(writer.Complete(), context).Fields, Is.Empty); + }); + } + + [TestCase(null, "Items", "sourceName")] + [TestCase("", "Items", "sourceName")] + [TestCase(" \t", "Items", "sourceName")] + [TestCase("Values", null, "destinationName")] + [TestCase("Values", "", "destinationName")] + [TestCase("Values", " \t", "destinationName")] + public void UpgradeCopy_RejectsInvalidNamesWithoutWriting(string? sourceName, string? destinationName, string parameterName) { + SerializationContext context = new(RegisterTypes()); + NetworkObjectUpgradeReader reader = ReadUpgradeFields(BuildObjectPayload(MemberIdentificationMode.Name, BuildNameField("Values", Int32(3))).AsSpan(2), context); + NetworkObjectUpgradeWriter writer = new(new BufferWriter(), reader, context); + + Assert.That(() => writer.Copy(sourceName!, destinationName!), Throws.InstanceOf().With.Property("ParamName").EqualTo(parameterName)); + Assert.That(ReadUpgradeFields(writer.Complete(), context).Fields, Is.Empty); + } + + [Test] + public void UpgradeCopy_RequiresASourceReader() { + SerializationContext context = new(RegisterTypes()); + NetworkObjectUpgradeWriter writer = new(new BufferWriter(), null, context); + + Assert.That(() => writer.Copy("Values", "Items"), Throws.InvalidOperationException.With.Message.EqualTo("Copy requires a source upgrade reader.")); + Assert.That(ReadUpgradeFields(writer.Complete(), context).Fields, Is.Empty); + } + + [TestCase("Missing")] + [TestCase("values")] + public void UpgradeCopy_RequiresAnOrdinalSourceMatch(string sourceName) { + SerializationContext context = new(RegisterTypes()); + NetworkObjectUpgradeReader reader = ReadUpgradeFields(BuildObjectPayload(MemberIdentificationMode.Name, BuildNameField("Values", Int32(3))).AsSpan(2), context); + NetworkObjectUpgradeWriter writer = new(new BufferWriter(), reader, context); + + Assert.Multiple(() => { + Assert.That(() => writer.Copy(sourceName, "Items"), Throws.TypeOf().With.Message.EqualTo($"Upgrade payload does not contain a field named '{sourceName}'.")); + Assert.That(() => reader.Get(sourceName), Throws.TypeOf()); + Assert.That(reader.TryGet(sourceName, out _), Is.False); + Assert.That(ReadUpgradeFields(writer.Complete(), context).Fields, Is.Empty); + }); + } + + [Test] + public void UpgradeCopy_UsesTheLastDuplicateAndDoesNotRemoveOrReplaceFields() { + SerializationContext context = new(RegisterTypes()); + NetworkObjectUpgradeReader reader = ReadUpgradeFields(BuildObjectPayload(MemberIdentificationMode.Name, + BuildNameField("Values", Int32(1)), + BuildNameField("values", Int32(2)), + BuildNameField("Values", Int32(3))).AsSpan(2), context); + NetworkObjectUpgradeWriter writer = new(new BufferWriter(), reader, context); + writer.CopyExcept("Values"); + writer.Write("Items", 0); + writer.Copy("Values", "Items"); + writer.Copy("values", "Éléments"); + writer.Copy("Values", "Values"); + NetworkObjectUpgradeReader output = ReadUpgradeFields(writer.Complete(), context); + + Assert.Multiple(() => { + Assert.That(reader.Get("Values"), Is.EqualTo(3)); + Assert.That(reader.TryGet("Values", out int value), Is.True); + Assert.That(value, Is.EqualTo(3)); + Assert.That(reader.Fields, Has.Count.EqualTo(3)); + Assert.That(output.Fields.Select(field => field.Name), Is.EqualTo(new[] { "values", "Items", "Items", "Éléments", "Values" })); + Assert.That(output.Get("Items"), Is.EqualTo(3)); + Assert.That(output.Get("Éléments"), Is.EqualTo(2)); + Assert.That(output.Get("Values"), Is.EqualTo(3)); + Assert.That(output.Fields[1].Value.ToArray(), Is.EqualTo(Int32(0))); + Assert.That(output.Fields[2].Value.ToArray(), Is.EqualTo(reader.Fields[2].Value.ToArray())); + }); + } + + [Test] + public void UpgradeCopy_CopiesOpaqueEmptyValuesFromTheReaderSnapshot() { + SerializationContext context = new(RegisterTypes()); + byte[] payload = BuildObjectPayload(MemberIdentificationMode.Name, BuildNameField("Empty", [])); + NetworkObjectUpgradeReader reader = ReadUpgradeFields(payload.AsSpan(2), context); + Array.Fill(payload, (byte)0xff); + NetworkObjectUpgradeWriter writer = new(new BufferWriter(), reader, context); + writer.Copy("Empty", "Renamed"); + NetworkObjectUpgradeReader output = ReadUpgradeFields(writer.Complete(), context); + + Assert.That(output.Fields.Single().Name, Is.EqualTo("Renamed")); + Assert.That(output.Fields.Single().Value.IsEmpty, Is.True); + } + + [Test] + public void UpgradeCopy_RejectsWritesAfterCompletionAndKeepsCompletionIdempotent() { + SerializationContext context = new(RegisterTypes()); + NetworkObjectUpgradeReader reader = ReadUpgradeFields(BuildObjectPayload(MemberIdentificationMode.Name, BuildNameField("Values", Int32(3))).AsSpan(2), context); + NetworkObjectUpgradeWriter writer = new(new BufferWriter(), reader, context); + writer.Copy("Values", "Items"); + byte[] completed = writer.Complete(); + + Assert.Multiple(() => { + Assert.That(() => writer.Copy("Values", "Later"), Throws.InvalidOperationException.With.Message.Contains("completed")); + Assert.That(() => writer.Write("Later", 5), Throws.InvalidOperationException.With.Message.Contains("completed")); + Assert.That(() => writer.CopyExcept(), Throws.InvalidOperationException.With.Message.Contains("completed")); + Assert.That(writer.Complete(), Is.EqualTo(completed)); + Assert.That(ReadUpgradeFields(completed, context).Fields.Single().Name, Is.EqualTo("Items")); + }); + } + + [Test] + public void UpgradeCopy_EnforcesTheExistingFieldCountLimit() { + SerializationContext context = new(RegisterTypes()); + NetworkObjectUpgradeReader reader = ReadUpgradeFields(BuildObjectPayload(MemberIdentificationMode.Name, BuildNameField("V", Int32(3))).AsSpan(2), context); + BufferWriter buffer = new(); + NetworkObjectUpgradeWriter writer = new(buffer, reader, context); + for (int index = 0; index < ushort.MaxValue; index++) { + writer.Copy("V", "V"); + } + int fullLength = buffer.GetWrittenSpan().Length; + + Assert.Multiple(() => { + Assert.That(() => writer.Copy("V", "Extra"), Throws.InvalidOperationException.With.Message.EqualTo("Cannot write more than 65535 fields.")); + Assert.That(() => writer.Write("Extra", 5), Throws.InvalidOperationException.With.Message.Contains("65535 fields")); + Assert.That(() => writer.CopyExcept(), Throws.InvalidOperationException.With.Message.Contains("65535 fields")); + Assert.That(buffer.GetWrittenSpan().Length, Is.EqualTo(fullLength)); + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(writer.Complete().AsSpan(1)), Is.EqualTo(ushort.MaxValue)); + }); + } + + private static TypeCatalogue RegisterUpgradeCopyTypes() { + return RegisterTypes(typeof(LegacyUpgradeCollectionsState), typeof(RenamedUpgradeCollectionsState), + typeof(UnchangedUpgradeCollectionsState), typeof(UpgradeNestedCollectionState), typeof(DirtyChildState)); + } + + private static UpgradeNestedCollectionState CreateUpgradeNestedCollection(int value) { + UpgradeNestedCollectionState result = new(); + result.Values.Add(null); + result.Values.Add(value); + result.Labels.Add(value, "nested 猫"); + return result; + } + + private static NetworkObjectUpgradeReader ReadUpgradeFields(ReadOnlySpan payload, SerializationContext context) { + Assert.That(NetworkObjectUpgradeReader.TryCreate(payload, context, out NetworkObjectUpgradeReader reader), Is.True); + return reader; + } + + private static void AssertRenamedUpgradeBytesArePreserved(byte[] payload, TypeCatalogue catalogue) { + SerializationContext context = new(catalogue); + NetworkObjectUpgradeReader reader = ReadUpgradeFields(payload.AsSpan(2), context); + NetworkObjectUpgradeWriter writer = new(new BufferWriter(), reader, context); + RenamedUpgradeCollectionsState.UpgradeToVersion2(reader, writer); + NetworkObjectUpgradeReader output = ReadUpgradeFields(writer.Complete(), context); + (string Source, string Destination)[] fields = [ + ("Marker", "Marker"), ("Values", "Items"), ("Labels", "Names"), ("OptionalValue", "OptionalCount"), + ("Child", "Detail"), ("NullableValues", "OptionalItems"), ("Children", "NestedItems"), ("ChildLookup", "NestedLookup") + ]; + + Assert.That(output.Fields.Select(field => field.Name), Is.EquivalentTo(fields.Select(field => field.Destination))); + foreach ((string sourceName, string destinationName) in fields) { + Assert.That(output.Fields.Single(field => field.Name == destinationName).Value.ToArray(), + Is.EqualTo(reader.Fields.Single(field => field.Name == sourceName).Value.ToArray()), destinationName); + } + } +} diff --git a/Cat.Network/NetworkObjectUpgradeReader.cs b/Cat.Network/NetworkObjectUpgradeReader.cs index 58888e7..99008da 100644 --- a/Cat.Network/NetworkObjectUpgradeReader.cs +++ b/Cat.Network/NetworkObjectUpgradeReader.cs @@ -18,11 +18,7 @@ private NetworkObjectUpgradeReader(IReadOnlyList fiel public IReadOnlyList Fields { get; } public T Get(string name) { - ArgumentException.ThrowIfNullOrWhiteSpace(name); - if (!fieldsByName.TryGetValue(name, out NetworkObjectUpgradeField field)) { - throw new KeyNotFoundException($"Upgrade payload does not contain a field named '{name}'."); - } - + NetworkObjectUpgradeField field = GetField(name); return NetworkObjectUpgradeCodec.Deserialize(field.Value.Span, context); } @@ -37,6 +33,15 @@ public bool TryGet(string name, out T value) { return true; } + internal NetworkObjectUpgradeField GetField(string name) { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + if (!fieldsByName.TryGetValue(name, out NetworkObjectUpgradeField field)) { + throw new KeyNotFoundException($"Upgrade payload does not contain a field named '{name}'."); + } + + return field; + } + public static bool TryCreate(ReadOnlySpan data, SerializationContext context, out NetworkObjectUpgradeReader reader) { reader = null!; if (data.Length < 3) { diff --git a/Cat.Network/NetworkObjectUpgradeWriter.cs b/Cat.Network/NetworkObjectUpgradeWriter.cs index 9ffec83..34483b6 100644 --- a/Cat.Network/NetworkObjectUpgradeWriter.cs +++ b/Cat.Network/NetworkObjectUpgradeWriter.cs @@ -16,6 +16,29 @@ public NetworkObjectUpgradeWriter(BufferWriter writer, NetworkObjectUpgradeReade fieldCountRange = writer.Reserve(2); } + /// Appends a source field's encoded value under the destination name without decoding it. + /// + /// Source names use ordinal comparison, and the last occurrence of a duplicate source name is copied. + /// This does not remove source fields or replace fields already written. To rename a field, exclude its + /// old name (and any destination field being replaced) from before copying it. + /// The value bytes are preserved exactly; element types and collection encodings are not converted. + /// + /// Either name is null, empty, or whitespace. + /// The source reader does not contain the source field. + /// + /// There is no source reader, the writer is completed, or the field count limit has been reached. + /// + public void Copy(string sourceName, string destinationName) { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceName); + ArgumentException.ThrowIfNullOrWhiteSpace(destinationName); + if (reader is null) { + throw new InvalidOperationException("Copy requires a source upgrade reader."); + } + + NetworkObjectUpgradeField field = reader.GetField(sourceName); + WriteRaw(destinationName, field.Value.Span); + } + public void CopyExcept(params string[] names) { if (reader is null) { throw new InvalidOperationException("CopyExcept requires a source upgrade reader."); diff --git a/README.md b/README.md index ca06dc1..c03accb 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Use `MemberSelectionMode.All` for full payloads and `MemberSelectionMode.Dirty` ## Version Upgrades -Set the current schema version on `[NetworkObject]`. Upgrade methods are static methods marked with `[UpgradeTo(version)]` and must upgrade exactly one version step at a time. +Set the current schema version on `[NetworkObject]`. Upgrade methods are static methods marked with `[UpgradeTo(version)]` and must upgrade exactly one version step at a time. All upgrade reads and writes must finish synchronously before the method returns; `async void` methods and deferred or background writes are not supported. ```csharp [NetworkObject(Version = 2)] @@ -130,7 +130,23 @@ Upgrade rules: - each `[UpgradeTo]` method moves from `version - 1` to `version` - missing sequential upgrade steps make deserialization fail - `CopyExcept(...)` copies unchanged old fields by name -- `Write(name, value)` writes fields in the new schema shape +- `Copy(sourceName, destinationName)` appends an existing source field's encoded value under the destination name +- `Get`, `TryGet`, and `Write(name, value)` decode or encode supported scalar, nullable, string, `Guid`, struct, and `NetworkObject` values; typed collection reads and writes are not supported + +Use `Copy` to rename a list or dictionary while preserving its serialized value bytes: + +```csharp +[UpgradeTo(2)] +private static void UpgradeToVersion2(NetworkObjectUpgradeReader reader, NetworkObjectUpgradeWriter writer) { + writer.CopyExcept("Values", "Labels"); + writer.Copy("Values", "Items"); + writer.Copy("Labels", "Names"); +} +``` + +`Copy` requires a source reader and a present source field, and both names must be nonempty and non-whitespace. Source lookup is ordinal and case-sensitive; if a name occurs more than once, the last occurrence is copied, as with `Get`. A missing source field throws `KeyNotFoundException`. Copying does not remove fields or replace fields already written: exclude the old names, and any destination fields being replaced, from `CopyExcept` when renaming. + +Raw copying does not decode or transform collection contents, change element types, or convert legacy wire encodings. The destination must accept the existing value encoding. Collection content transformations through typed `Get`/`Write` remain unsupported; encoding changes require a separate compatible conversion. Finish copying before returning from the upgrade method, since the generated serializer then completes the writer. ## Clone Behavior