refactor: std container migration stage 2 — the protected surface (#60)

CLAUDE.md

@@ -101,14 +101,36 @@ Release is what ships and exercises the backstops the asserts hide):
 ```bash
 docker run --rm -v "$PWD":/src:ro ubuntu:24.04 bash -c '
   apt-get update -qq && apt-get install -y -qq build-essential cmake libssl-dev git ca-certificates
-  cp -r /src /work && cd /work
   for cfg in Debug Release; do
-    cmake -B /b-$cfg -S . -DCMAKE_BUILD_TYPE=$cfg -DMAFIANET_BUILD_TESTS=ON
+    cmake -B /b-$cfg -S /src -DCMAKE_BUILD_TYPE=$cfg -DMAFIANET_BUILD_TESTS=ON
     cmake --build /b-$cfg --parallel "$(nproc)"
     ctest --test-dir /b-$cfg --output-on-failure --timeout 600 || exit 1
   done'
 ```
 
+Build straight from the read-only mount with `-S /src`; **do not** `cp -r /src /work`
+first. A local `build/` directory is a few hundred MB, the copy pulls it in, and once
+the container's writable layer fills up it truncates files *silently* -- the symptom
+is gcc reporting "unterminated #if" in a source file that compiles fine everywhere
+else. CMake never writes into the source tree, so there is nothing to copy for.
+
+To check for leaks and memory errors, add a sanitizer to the same invocation and run
+the binaries directly (LeakSanitizer is Linux-only; macOS ASAN has no leak
+detection):
+
+```bash
+docker run --rm -v "$PWD":/src:ro ubuntu:24.04 bash -c '
+  apt-get update -qq && apt-get install -y -qq build-essential cmake libssl-dev git ca-certificates
+  cmake -B /b -S /src -DCMAKE_BUILD_TYPE=Debug -DMAFIANET_BUILD_TESTS=ON \
+    -DCMAKE_CXX_FLAGS="-fsanitize=address -fno-omit-frame-pointer -g" \
+    -DCMAKE_C_FLAGS="-fsanitize=address -fno-omit-frame-pointer -g" \
+    -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address" \
+    -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=address"
+  cmake --build /b --parallel "$(nproc)"
+  ASAN_OPTIONS=detect_leaks=1 /b/Tests/UnitTests
+  ASAN_OPTIONS=detect_leaks=1 /b/Tests/IntegrationTests --gtest_filter="<Suite>.*"'
+```
+
 `Tests/Integration/MmsgBatchLiveTests.cpp` is the suite that actually fills batches
 (the rest of the suite sends ~1 datagram per tick and never crosses `MMSG_BATCH_MAX`).
 Keep its message counts above `MMSG_BATCH_MAX` or that coverage silently disappears.

DependentExtensions/Autopatcher/AutopatcherServer.h

@@ -35,6 +35,7 @@
 #include "mafianet/plugin_interface2.h"
 #include "mafianet/packet_priority.h"
 #include "mafianet/thread_pool.h"
+#include "mafianet/ds_queue.h"
 #include "mafianet/bit_stream.h"
 #include "mafianet/string.h"
 #include "mafianet/file_list.h"

DependentExtensions/Lobby2/Lobby2Server.h

@@ -21,6 +21,7 @@
 #include "Lobby2Plugin.h"
 #include "mafianet/ds_ordered_list.h"
 #include "mafianet/thread_pool.h"
+#include "mafianet/ds_queue.h"
 #include "Lobby2Presence.h"
 
 //class PostgreSQLInterface;

Source/include/mafianet/ds_table.h

@@ -25,6 +25,7 @@
 #include "mafianet/memory_override.h"
 #include "mafianet/export.h"
 #include "mafianet/string.h"
+#include <vector>
 
 #define _TABLE_MAX_COLUMN_NAME_LENGTH 64
 
@@ -330,11 +331,11 @@ namespace DataStructures
 		Table& operator = ( const Table& input );
 
 	protected:
-		Table::Row* AddRowColumns(unsigned rowId, Row *row, DataStructures::List<unsigned> columnIndices);
+		Table::Row* AddRowColumns(unsigned rowId, Row *row, std::vector<unsigned> columnIndices);
 
 		void DeleteRow(Row *row);
 
-		void QueryRow(DataStructures::List<unsigned> &inclusionFilterColumnIndices, DataStructures::List<unsigned> &columnIndicesToReturn, unsigned key, Table::Row* row, FilterQuery *inclusionFilters, Table *result);
+		void QueryRow(std::vector<unsigned> &inclusionFilterColumnIndices, std::vector<unsigned> &columnIndicesToReturn, unsigned key, Table::Row* row, FilterQuery *inclusionFilters, Table *result);
 
 		std::map<unsigned, Row*> rows;
 

Source/include/mafianet/message_filter.h

@@ -27,6 +27,8 @@
 #include "mafianet/plugin_interface2.h"
 #include "mafianet/ds_ordered_list.h"
 #include "mafianet/ds_hash.h"
+#include <unordered_map>
+#include <vector>
 #include "mafianet/export.h"
 
 /// MessageIdentifier (ID_*) values shoudln't go higher than this.  Change it if you do.
@@ -188,9 +190,28 @@ class RAK_DLL_EXPORT MessageFilter : public PluginInterface2
 	FilterSet* GetFilterSetByID(int filterSetID);
 	void OnInvalidMessage(FilterSet *filterSet, AddressOrGUID systemAddress, unsigned char messageID);
 
-	DataStructures::OrderedList<int, FilterSet*, FilterSetComp> filterList;
+	// Kept sorted ascending by FilterSet::filterSetID and searched with std::lower_bound,
+	// matching the OrderedList it replaces. GetFilterSetIDByIndex publishes this index.
+	std::vector<FilterSet *> filterList;
+
 	// Change to guid
-	DataStructures::Hash<AddressOrGUID, FilteredSystem, 2048, AddressOrGUID::ToInteger> systemList;
+	// Same hash and same equality as the DataStructures::Hash this replaces, so behaviour is
+	// unchanged -- including a pre-existing quirk worth knowing about: AddressOrGUID::operator==
+	// matches when EITHER the guid or the address matches, while ToInteger hashes on the guid
+	// when one is set and the address otherwise. Two keys that compare equal can therefore hash
+	// differently and miss each other. That is inherited deliberately; changing key equality
+	// would change which systems a filter set matches.
+	struct AddressOrGUIDKeyHash
+	{
+		size_t operator()(const AddressOrGUID &aog) const
+		{
+			return (size_t) AddressOrGUID::ToInteger(aog);
+		}
+	};
+	std::unordered_map<AddressOrGUID, FilteredSystem, AddressOrGUIDKeyHash> systemList;
+
+	// Position of filterSetID in filterList, or of where it would be inserted.
+	unsigned int IndexForFilterSetID(int filterSetID, bool *found) const;
 
 	int autoAddNewConnectionsToFilter;
 	MafiaNet::Time whenLastTimeoutCheck;

Source/include/mafianet/packetized_tcp.h

@@ -26,7 +26,9 @@
 
 #include "mafianet/tcp_interface.h"
 #include "mafianet/ds_byte_queue.h"
-#include "mafianet/ds_map.h"
+
+#include <deque>
+#include <map>
 
 namespace MafiaNet
 {
@@ -71,17 +73,23 @@ class RAK_DLL_EXPORT PacketizedTCP : public TCPInterface
 
 protected:
 	void ClearAllConnections(void);
+	/// Frees every reassembled message still waiting for Receive, and empties the queue.
+	void ClearWaitingPackets(void);
 	void RemoveFromConnectionList(const SystemAddress &sa);
 	void AddToConnectionList(const SystemAddress &sa);
 	void PushNotificationsToQueues(void);
 	Packet *ReturnOutgoingPacket(void);
 
 	// A single TCP recieve may generate multiple split packets. They are stored in the waitingPackets list until Receive is called
-	DataStructures::Queue<Packet*> waitingPackets;
-	DataStructures::Map<SystemAddress, DataStructures::ByteQueue *> connections;
+	std::deque<Packet*> waitingPackets;
+
+	// Per-connection reassembly buffers, keyed by address. Ordered by
+	// SystemAddress::operator<, as DataStructures::Map was; nothing depends on the
+	// order, only on lookup by address.
+	std::map<SystemAddress, DataStructures::ByteQueue *> connections;
 
 	// Mirrors single producer / consumer, but processes them in Receive() before returning to user
-	DataStructures::Queue<SystemAddress> _newIncomingConnections, _lostConnections, _failedConnectionAttempts, _completedConnectionAttempts;
+	std::deque<SystemAddress> _newIncomingConnections, _lostConnections, _failedConnectionAttempts, _completedConnectionAttempts;
 };
 
 } // namespace MafiaNet

Source/include/mafianet/peer.h

@@ -42,6 +42,9 @@
 #include "mafianet/lockless_types.h"
 #include "mafianet/ds_queue.h"
 
+#include <deque>
+#include <vector>
+
 namespace MafiaNet {
 /// Forward declarations
 class HuffmanEncodingTree;
@@ -970,11 +973,11 @@ class RAK_DLL_EXPORT RakPeer : public RakPeerInterface, public RNS2EventHandler
 #endif
 
 	//DataStructures::List<DataStructures::List<MemoryBlock>* > automaticVariableSynchronizationList;
-	DataStructures::List<BanStruct*> banList;
+	std::vector<BanStruct*> banList;
 	// Threadsafe, and not thread safe
 	DataStructures::List<PluginInterface2*> pluginListTS, pluginListNTS;
 
-	DataStructures::Queue<RequestedConnectionStruct*> requestedConnectionQueue;
+	std::deque<RequestedConnectionStruct*> requestedConnectionQueue;
 	SimpleMutex requestedConnectionQueueMutex;
 
 	// void RunMutexedUpdateCycle(void);
@@ -1012,9 +1015,9 @@ class RAK_DLL_EXPORT RakPeer : public RakPeerInterface, public RNS2EventHandler
 
 	// DataStructures::ThreadsafeAllocatingQueue<RNS2RecvStruct> bufferedPackets;
 
-	DataStructures::Queue<RNS2RecvStruct*> bufferedPacketsFreePool;
+	std::deque<RNS2RecvStruct*> bufferedPacketsFreePool;
 	MafiaNet::SimpleMutex bufferedPacketsFreePoolMutex;
-	DataStructures::Queue<RNS2RecvStruct*> bufferedPacketsQueue;
+	std::deque<RNS2RecvStruct*> bufferedPacketsQueue;
 	MafiaNet::SimpleMutex bufferedPacketsQueueMutex;
 
 	virtual void DeallocRNS2RecvStruct(RNS2RecvStruct *s, const char *file, unsigned int line);
@@ -1027,7 +1030,7 @@ class RAK_DLL_EXPORT RakPeer : public RakPeerInterface, public RNS2EventHandler
 	{
 		SocketQueryOutput() {}
 		~SocketQueryOutput() {}
-		DataStructures::List<RakNetSocket2* > sockets;
+		std::vector<RakNetSocket2*> sockets;
 	};
 
 	DataStructures::ThreadsafeAllocatingQueue<SocketQueryOutput> socketQueryOutput;
@@ -1060,7 +1063,7 @@ class RAK_DLL_EXPORT RakPeer : public RakPeerInterface, public RNS2EventHandler
 	bool trackFrequencyTable;
 
 	// Smart pointer so I can return the object to the user
-	DataStructures::List<RakNetSocket2* > socketList;
+	std::vector<RakNetSocket2*> socketList;
 	void DerefAllSockets(void);
 	unsigned int GetRakNetSocketFromUserConnectionSocketIndex(unsigned int userIndex) const;
 	// Used for RPC replies
@@ -1095,7 +1098,7 @@ class RAK_DLL_EXPORT RakPeer : public RakPeerInterface, public RNS2EventHandler
 	bool (*incomingDatagramEventHandler)(RNS2RecvStruct *);
 
 	// Systems in this list will not go through the secure connection process, even when secure connections are turned on. Wildcards are accepted.
-	DataStructures::List<MafiaNet::RakString> securityExceptionList;
+	std::vector<MafiaNet::RakString> securityExceptionList;
 
 	SystemAddress ipList[ MAXIMUM_NUMBER_OF_INTERNAL_IDS ];
 
@@ -1112,7 +1115,7 @@ class RAK_DLL_EXPORT RakPeer : public RakPeerInterface, public RNS2EventHandler
 	DataStructures::MemoryPool<Packet> packetAllocationPool;
 
 	SimpleMutex packetReturnMutex;
-	DataStructures::Queue<Packet*> packetReturnQueue;
+	std::deque<Packet*> packetReturnQueue;
 	Packet *AllocPacket(unsigned dataSize, const char *file, unsigned int line);
 	Packet *AllocPacket(unsigned dataSize, unsigned char *data, const char *file, unsigned int line);
 

Source/include/mafianet/reliability_layer.h

@@ -49,6 +49,8 @@
 #define INCLUDE_TIMESTAMP_WITH_DATAGRAMS 1
 #else
 #include "mafianet/cc_sliding_window.h"
+#include <deque>
+#include <vector>
 #define INCLUDE_TIMESTAMP_WITH_DATAGRAMS 0
 #endif
 
@@ -408,7 +410,7 @@ class ReliabilityLayer//<ReliabilityLayer>
 	// Used ONLY for MafiaNet::Reliability::ReliableOrdered
 	// MafiaNet::Reliability::ReliableSequenced just returns the newest one
 	// DataStructures::List<DataStructures::LinkedList<InternalPacket*>*> orderingList;
-	DataStructures::Queue<InternalPacket*> outputQueue;
+	std::deque<InternalPacket *> outputQueue;
 	int splitMessageProgressInterval;
 	CCTimeType unreliableTimeout;
 
@@ -430,7 +432,9 @@ class ReliabilityLayer//<ReliabilityLayer>
 	// Queue length is programmatically restricted to DATAGRAM_MESSAGE_ID_ARRAY_LENGTH
 	// This is essentially an O(1) lookup to get a DatagramHistoryNode given an index
 	// datagramHistory holds a linked list of MessageNumberNode. Each MessageNumberNode refers to one element in resendList which can be cleared on an ack.
-	DataStructures::Queue<DatagramHistoryNode> datagramHistory;
+	// A sliding window: index 0 is datagram number datagramHistoryPopCount, and lookups index
+	// it by (datagramNumber - datagramHistoryPopCount) in wrapping uint24_t arithmetic.
+	std::deque<DatagramHistoryNode> datagramHistory;
 	DataStructures::MemoryPool<MessageNumberNode> datagramHistoryMessagePool;
 
 	struct UnreliableWithAckReceiptNode
@@ -443,7 +447,7 @@ class ReliabilityLayer//<ReliabilityLayer>
 		uint32_t sendReceiptSerial;
 		MafiaNet::TimeUS nextActionTime;
 	};
-	DataStructures::List<UnreliableWithAckReceiptNode> unreliableWithAckReceiptHistory;
+	std::vector<UnreliableWithAckReceiptNode> unreliableWithAckReceiptHistory;
 
 	void RemoveFromDatagramHistory(DatagramSequenceNumberType index);
 	MessageNumberNode* GetMessageNumberNodeByDatagramIndex(DatagramSequenceNumberType index, CCTimeType *timeSent);
@@ -487,7 +491,10 @@ class ReliabilityLayer//<ReliabilityLayer>
 //	double bytesInSendBuffer[MafiaNet::NUMBER_OF_PRIORITIES];
 
 
-    DataStructures::OrderedList<SplitPacketIdType, SplitPacketChannel*, SplitPacketChannelComp> splitPacketChannelList;
+	// Kept sorted by split-packet id and searched with std::lower_bound through
+	// IndexForSplitPacketId, matching the OrderedList it replaces. Several split messages can
+	// be in flight at once, so the keyed lookup decides which message a fragment belongs to.
+	std::vector<SplitPacketChannel *> splitPacketChannelList;
 
 	MessageNumberType sendReliableMessageNumberIndex;
 	MessageNumberType internalOrderIndex;
@@ -549,7 +556,10 @@ class ReliabilityLayer//<ReliabilityLayer>
 	/// If we get a packet number where (receivedPacketsBaseIndex-packetNumber) is less than half the range of receivedPacketsBaseIndex then it is a duplicate
 	/// Otherwise, it is a duplicate packet (and ignore it).
 	// DataStructures::Queue<CCTimeType> hasReceivedPacketQueue;
-	DataStructures::Queue<bool> hasReceivedPacketQueue;
+	// Sliding window of holes, indexed from the head. std::deque frees blocks as it drains, so
+	// the predecessor's manual AllocationSize()/Compress() compaction is gone -- that existed
+	// only because its ring buffer never shrank on its own.
+	std::deque<bool> hasReceivedPacketQueue;
 	DatagramSequenceNumberType receivedPacketsBaseIndex;
 	bool resetReceivedPackets;
 
@@ -586,7 +596,7 @@ class ReliabilityLayer//<ReliabilityLayer>
 		//	SystemAddress systemAddress;
 		unsigned int extraSocketOptions;
 	};
-	DataStructures::Queue<DataAndTime*> delayList;
+	std::deque<DataAndTime *> delayList;
 
 	// Internet simulator
 	double packetloss;
@@ -623,15 +633,23 @@ class ReliabilityLayer//<ReliabilityLayer>
 	void AddToListTail(InternalPacket *internalPacket, bool modifyUnacknowledgedBytes);
 	void PopListHead(bool modifyUnacknowledgedBytes);
 	bool IsResendQueueEmpty(void) const;
-	void SortSplitPacketList(DataStructures::List<InternalPacket*> &data, unsigned int leftEdge, unsigned int rightEdge) const;
+	void SortSplitPacketList(std::vector<InternalPacket *> &data, unsigned int leftEdge, unsigned int rightEdge) const;
+
+	// Position of a split-packet id in splitPacketChannelList, or where it would be inserted.
+	unsigned int IndexForSplitPacketId(SplitPacketIdType id, bool *found) const;
+	// Sorted insert; returns the index, or (unsigned)-1 if that id is already present, matching
+	// the OrderedList::Insert(assertOnDuplicate=true) it replaces. The id is passed in rather
+	// than read back out of the channel: at the call site the channel's split packet list has
+	// not been populated yet, so its own id is not readable there.
+	unsigned int InsertSplitPacketChannel(SplitPacketIdType id, SplitPacketChannel *channel);
 	void SendACKs(RakNetSocket2 *s, SystemAddress &systemAddress, CCTimeType time, RakNetRandom *rnr, BitStream &updateBitStream);
 
-	DataStructures::List<InternalPacket*> packetsToSendThisUpdate;
-	DataStructures::List<bool> packetsToDeallocThisUpdate;
+	std::vector<InternalPacket *> packetsToSendThisUpdate;
+	std::vector<bool> packetsToDeallocThisUpdate;
 	// boundary is in packetsToSendThisUpdate, inclusive
-	DataStructures::List<unsigned int> packetsToSendThisUpdateDatagramBoundaries;
-	DataStructures::List<bool> datagramsToSendThisUpdateIsPair;
-	DataStructures::List<unsigned int> datagramSizesInBytes;
+	std::vector<unsigned int> packetsToSendThisUpdateDatagramBoundaries;
+	std::vector<bool> datagramsToSendThisUpdateIsPair;
+	std::vector<unsigned int> datagramSizesInBytes;
 	BitSize_t datagramSizeSoFar;
 	BitSize_t allDatagramSizesSoFar;
 	double totalUserDataBytesAcked;