diff --git a/integration_test/test_filters.py b/integration_test/test_filters.py index 6ee76b7e..a87b630a 100644 --- a/integration_test/test_filters.py +++ b/integration_test/test_filters.py @@ -115,6 +115,45 @@ def repeat_s(filters: str) -> None: assert micro_received == len(EXPECTED_COUNTS) +def postprocess() -> None: + instance = Instance({Operator.F_INIT: ["macro", "meso", "micro"]}) + + while instance.reuse_instance(): + macro = instance.receive("macro") + meso = instance.receive("meso") + micro = instance.receive("micro") + + print("Received:", macro.data, meso.data, micro.data) + if macro.data[1] == 0: + # meso and micro didn't run in this iteration, so we get an empty message + assert meso.data is None + assert micro.data is None + else: + assert meso.data[-1] == macro.data[-1] - 1 + assert micro.data[-1] == 1 + + +def combined(filters: str) -> None: + is_padded = filters.split()[-1] == "pad" + + instance = Instance({Operator.F_INIT: ["trigger"], Operator.S: ["in"]}) + + reused = 0 + while instance.reuse_instance(): + instance.receive("trigger") + + last_value = None if reused == 0 else ["macro", reused, "meso", reused - 1] + for i in range(3): + msg = instance.receive("in") + if i and is_padded: + assert msg.data is None + else: + assert msg.data == last_value + reused += 1 + + assert reused == 4 + + config = """ ymmsl_version: v0.2 models: @@ -149,19 +188,35 @@ def repeat_s(filters: str) -> None: ports: f_init: macro meso micro implementation: pico + postprocess: + description: Postprocessing of final actor outputs + ports: + f_init: macro meso micro + implementation: postprocess + combined: + description: Receives meso.out with a combined reducer and repeater filter + ports: + f_init: trigger + s: in + implementation: combined conduits: macro.out: - meso.in - {filters} pico.macro - {filters} repeat_s.macro + - postprocess.macro + - combined.trigger meso.out: - micro.in - repeat pico.meso - repeat_s.meso - repeat repeat_s.repeated_meso + - last postprocess.meso + - last {combined_filter} combined.in micro.out: - pico.micro - repeat_s.micro + - last last postprocess.micro """ @@ -173,8 +228,14 @@ def test_repeater_filters(tmp_path, filters): "micro": ("python", micro), "repeat_s": ("python", repeat_s, filters), "pico": ("python", pico, filters), + "postprocess": ("python", postprocess), + "combined": ("python", combined, filters), } - run_manager_with_actors(config.format(filters=filters), tmp_path, actors) + run_manager_with_actors( + config.format(filters=filters, combined_filter=filters.split()[-1]), + tmp_path, + actors, + ) @skip_if_python_only @@ -186,8 +247,14 @@ def test_repeater_filters_cpp(tmp_path, filters): "micro": ("python", micro), "repeat_s": ("cpp", "conduit_filters_test", "repeat_s", filters), "pico": ("cpp", "conduit_filters_test", "pico", filters), + "postprocess": ("python", postprocess), + "combined": ("python", combined, filters), } - run_manager_with_actors(config.format(filters=filters), tmp_path, actors) + run_manager_with_actors( + config.format(filters=filters, combined_filter=filters.split()[-1]), + tmp_path, + actors, + ) checkpoint_config = """ diff --git a/src/cpp/libmuscle/communicator.cpp b/src/cpp/libmuscle/communicator.cpp index a5b14534..68379c98 100644 --- a/src/cpp/libmuscle/communicator.cpp +++ b/src/cpp/libmuscle/communicator.cpp @@ -96,8 +96,9 @@ std::vector Communicator::get_locations() const { } void Communicator::set_peer_info(PeerInfo const & peer_info) { + timeline_ = manager_.get_timeline(); peer_info_ = peer_info; - timeline_manager_ = std::make_unique(port_manager_); + timeline_manager_ = std::make_unique(port_manager_, timeline_.get()); prepare_conduit_filters_(); } @@ -161,8 +162,16 @@ void Communicator::send_message( if (message.has_next_timestamp()) mpp_message.next_timestamp = message.next_timestamp(); - - auto message_bytes = mpp_message.encoded(); + + std::vector message_bytes; + auto peer_port = recv_endpoint.kernel + recv_endpoint.port; + if (outgoing_timeline_length_.count(peer_port) > 0) { + message_bytes = apply_reduce_filters_(peer_port, std::move(mpp_message)); + if (message_bytes.empty()) + continue; + } else { + message_bytes = mpp_message.encoded(); + } profile_event.message_size = message_bytes.size(); server_.deposit(recv_endpoint.ref(), std::move(message_bytes)); } @@ -177,6 +186,57 @@ void Communicator::send_message( port.set_closed(slot); } +std::vector Communicator::apply_reduce_filters_( + ymmsl::Reference const & peer_port, MPPMessage && message) { + message.message_number = -1; // GH#411: Disabled checkpointing for reducer filter + + auto reduced_count = outgoing_timeline_length_.at(peer_port); + + if (!is_milestone(message.data)) { + // Reduce the message iteration count to match with the timeline we send to + message.iteration.resize(reduced_count); + auto it = reducer_cache_.find(message.receiver); + if (it != reducer_cache_.end()) + reducer_cache_.erase(it); // Remove existing entry + reducer_cache_.emplace(message.receiver, message); + log_debug("Message for ", message.receiver, " stored in cache"); + return {}; + } + + // Decide whether to send the milestone, ignore it, or send a cached message. + std::size_t n_milestone = message.iteration.size(); + if (n_milestone < reduced_count) { + // Milestone from ancestor timeline: send it + return message.encoded(); + } else if (n_milestone == reduced_count) { + // This is the target timeline after reduce filters applied: we need to + // send the cached message (or make up an empty one) and discard the + // milestone: + auto it = reducer_cache_.find(message.receiver); + if (it == reducer_cache_.end()) { + log_info( + "No cached message available to send because this instance did ", + "not run. Sending an empty message to ", message.receiver, " instead."); + return MPPMessage( + message.sender, message.receiver, message.port_length, + message.timestamp, message.next_timestamp, message.settings_overlay, + message.message_number, Data(), message.iteration + ).encoded(); + } + + assert(it->second.iteration == message.iteration); + log_debug("Sending cached message to ", message.receiver); + auto encoded = it->second.encoded(); + reducer_cache_.erase(it); + return encoded; + } else { + log_debug( + "Ignored milestone for ", message.receiver, + " because of LAST filters."); + return {}; + } +} + Communicator::FInitCacheType Communicator::pre_receive() { assert(timeline_manager_); auto finished_iteration = timeline_manager_->start_reuse_iteration(); @@ -376,7 +436,10 @@ MPPMessage Communicator::receive_message_( } int expected_message_number = port.get_num_messages(slot); - if (expected_message_number != mpp_message.message_number) { + if ( + mpp_message.message_number >= 0 // GH#411: negative for reducer filters + && expected_message_number != mpp_message.message_number + ) { if (expected_message_number - 1 == mpp_message.message_number and port.is_resuming(slot)) { log_debug("Discarding received message on ", port_and_slot, @@ -560,7 +623,21 @@ void Communicator::prepare_conduit_filters_() { } } - // TODO: reducer filters + // Reducer filters + for (auto op : {Operator::O_I, Operator::O_F}) { + for (Port const & port : port_manager_.get_connected_ports(op, {})) { + for (auto & peer_port : peer_info_.get().get_peer_ports(port.name)) { + auto & filters = peer_info_.get().get_filters_for_receiver(peer_port); + // Count the reducer filters, receiving component handles repeaters + auto n_reducers = std::count_if( + filters.begin(), filters.end(), ::ymmsl::is_reducer); + if (n_reducers > 0) { + std::size_t reduced_count = timeline_.get().size() + port.timeline.size() - n_reducers; + outgoing_timeline_length_.emplace(peer_port, reduced_count); + } + } + } + } } diff --git a/src/cpp/libmuscle/communicator.hpp b/src/cpp/libmuscle/communicator.hpp index 3ab51e9f..62c72110 100644 --- a/src/cpp/libmuscle/communicator.hpp +++ b/src/cpp/libmuscle/communicator.hpp @@ -170,6 +170,19 @@ class Communicator { Optional slot = {} ); + /** Apply reduce filters to a message sent on a conduit with reduce filters. + * + * User-provided messages (through instance.send()) will be stored (overwriting any + * existing message). For milestones this method decides if the milestone should be + * sent, or a cached message, or nothing at all. + * + * @param peer_port Peer port (component + port) to send to. + * @param message MPPMessage to be checked. + * @return The encoded MPPMessage to send, or an empty vector if we do not need to send anything. + */ + std::vector apply_reduce_filters_( + ymmsl::Reference const & peer_port, MPPMessage && message); + ymmsl::Reference instance_id_() const; MPPClient & get_client_(ymmsl::Reference const & instance); @@ -242,10 +255,26 @@ class Communicator { Optional peer_info_; double receive_timeout_; std::unique_ptr timeline_manager_; + Optional timeline_; PortManager::PortReferences pre_receive_ports_; std::unordered_map> repeat_filters_; MPPCacheType message_cache_; + + /** Size of IterationCount, after applying the reducer filters, per peer port. + * + * Keys are references to peer ports: ``component + port``. The outgoing + * timeline length is the size of the IterationCount after applying the + * reducer filters and determines in which (parent) timeline these messages + * are sent. + * If our timeline is ":macro:micro" then: + * - outgoing_timeline_length = 0: send on the root (":") timeline + * - outgoing_timeline_length = 1: send on the ":macro" timeline + * - outgoing_timeline_length = 2: send on the ":macro:micro" timeline + */ + std::unordered_map<::ymmsl::Reference, std::size_t> outgoing_timeline_length_; + /** Message cache for reducer filters */ + MPPCacheType reducer_cache_; }; } } diff --git a/src/cpp/libmuscle/tests/mocks/mock_timeline_manager.hpp b/src/cpp/libmuscle/tests/mocks/mock_timeline_manager.hpp index 403d09e3..3662a969 100644 --- a/src/cpp/libmuscle/tests/mocks/mock_timeline_manager.hpp +++ b/src/cpp/libmuscle/tests/mocks/mock_timeline_manager.hpp @@ -31,7 +31,8 @@ class MockTimelineManager : public MockClass { init_from_return_value(); } - explicit MockTimelineManager(PortManager const & port_manager) { + explicit MockTimelineManager( + PortManager const & port_manager, ::ymmsl::Timeline const & timeline) { init_from_return_value(); constructor(&port_manager); } diff --git a/src/cpp/libmuscle/tests/test_communicator_conduit_filters.cpp b/src/cpp/libmuscle/tests/test_communicator_conduit_filters.cpp index dec4527f..61d284aa 100644 --- a/src/cpp/libmuscle/tests/test_communicator_conduit_filters.cpp +++ b/src/cpp/libmuscle/tests/test_communicator_conduit_filters.cpp @@ -49,6 +49,7 @@ using libmuscle::_MUSCLE_IMPL_NS::encode_iteration; using libmuscle::_MUSCLE_IMPL_NS::Communicator; using libmuscle::_MUSCLE_IMPL_NS::DataConstRef; using libmuscle::_MUSCLE_IMPL_NS::IterationCount; +using libmuscle::_MUSCLE_IMPL_NS::Message; using libmuscle::_MUSCLE_IMPL_NS::MPPMessage; using libmuscle::_MUSCLE_IMPL_NS::MockLogger; using libmuscle::_MUSCLE_IMPL_NS::MockProfiler; @@ -65,6 +66,7 @@ using ymmsl::Conduit; using ymmsl::ConduitFilter; using ymmsl::Operator; using ymmsl::Port; +using ymmsl::Timeline; struct libmuscle_repeater_communicator : ::testing::TestWithParam @@ -81,6 +83,7 @@ struct libmuscle_repeater_communicator : port_manager_({}, {}) , communicator_("component", {}, port_manager_, profiler_, manager_) { + manager_.get_timeline.return_value = Timeline(":parent3:parent2:parent1"); std::string repeat_filter = GetParam(); PeerInfo peer_info( "component", @@ -107,6 +110,53 @@ struct libmuscle_repeater_communicator } }; +struct libmuscle_reducer_communicator + : ::testing::Test +{ + RESET_MOCKS(MockLogger, MockMMPClient, MockMPPClient, MockMPPServer, MockProfiler); + + MockProfiler profiler_; + MockMMPClient manager_; + + PortManager port_manager_; + Communicator communicator_; + + libmuscle_reducer_communicator() + : port_manager_({}, {}) + , communicator_("component", {}, port_manager_, profiler_, manager_) + { + manager_.get_timeline.return_value = Timeline(":parent"); + PeerInfo peer_info( + "component", + {}, + { + Conduit("parent.out", "component.init"), + Conduit("component.final", "parent.in"), + Conduit("component.final", "sibling.in2"), + // Reducer filter on O_I port + Conduit("component.out", "sibling.in", "last"), + // Reducer filter on O_F port + Conduit("component.final", "aunt.init", "last"), + // Double reducer filter on O_I port + Conduit("component.out", "uncle.init", "last last"), + }, + {{"parent", {}}, {"aunt", {}}, {"uncle", {}}, {"sibling", {}}}, + {{"parent", {}}, {"aunt", {}}, {"uncle", {}}, {"sibling", {}}}, + { + Port("init", Operator::F_INIT), + Port("out", Operator::O_I, Timeline("component")), + Port("final", Operator::O_F) + } + ); + port_manager_.connect_ports(peer_info); + communicator_.set_peer_info(peer_info); + } + + void TearDown() override { + communicator_.shutdown(); + } +}; + class IterationOrMilestone { public: IterationCount iteration; @@ -164,19 +214,20 @@ void mock_receive_messages( TEST_P(libmuscle_repeater_communicator, repeater_filters) { mock_receive_messages({ {"component.twicerepeated", { - I({}), M({}) + I({0}), M({}) }}, {"component.repeated", { - I({0}), I({1}), I({2}), M({}) + I({0, 0}), I({0, 1}), I({0, 2}), M({0}), M({}) }}, {"component.unfiltered", { - I({0, 0}), - I({0, 1}), - M({0}), + I({0, 0, 0}), + I({0, 0, 1}), + M({0, 0}), // parent is allowed to send 0 messages on its O_I port in an iteration - M({1}), - I({2, 0}), - M({2}), + M({0, 1}), + I({0, 2, 0}), + M({0, 2}), + M({0}), M({}) }} }); @@ -184,27 +235,27 @@ TEST_P(libmuscle_repeater_communicator, repeater_filters) { bool is_padded = GetParam() == "pad"; auto cache = communicator_.pre_receive(); - ASSERT_EQ(decode_iteration(cache.at("unfiltered").data()).get(), IterationCount({0, 0})); - ASSERT_EQ(decode_iteration(cache.at("repeated").data()).get(), IterationCount({0})); - ASSERT_EQ(decode_iteration(cache.at("twicerepeated").data()).get(), IterationCount({})); + ASSERT_EQ(decode_iteration(cache.at("unfiltered").data()).get(), IterationCount({0, 0, 0})); + ASSERT_EQ(decode_iteration(cache.at("repeated").data()).get(), IterationCount({0, 0})); + ASSERT_EQ(decode_iteration(cache.at("twicerepeated").data()).get(), IterationCount({0})); cache = communicator_.pre_receive(); - ASSERT_EQ(decode_iteration(cache.at("unfiltered").data()).get(), IterationCount({0, 1})); + ASSERT_EQ(decode_iteration(cache.at("unfiltered").data()).get(), IterationCount({0, 0, 1})); if (is_padded) { ASSERT_TRUE(cache.at("repeated").data().is_nil()); ASSERT_TRUE(cache.at("twicerepeated").data().is_nil()); } else { - ASSERT_EQ(decode_iteration(cache.at("repeated").data()).get(), IterationCount({0})); - ASSERT_EQ(decode_iteration(cache.at("twicerepeated").data()).get(), IterationCount({})); + ASSERT_EQ(decode_iteration(cache.at("repeated").data()).get(), IterationCount({0, 0})); + ASSERT_EQ(decode_iteration(cache.at("twicerepeated").data()).get(), IterationCount({0})); } cache = communicator_.pre_receive(); - ASSERT_EQ(decode_iteration(cache.at("unfiltered").data()).get(), IterationCount({2, 0})); - ASSERT_EQ(decode_iteration(cache.at("repeated").data()).get(), IterationCount({2})); + ASSERT_EQ(decode_iteration(cache.at("unfiltered").data()).get(), IterationCount({0, 2, 0})); + ASSERT_EQ(decode_iteration(cache.at("repeated").data()).get(), IterationCount({0, 2})); if (is_padded) { ASSERT_TRUE(cache.at("twicerepeated").data().is_nil()); } else { - ASSERT_EQ(decode_iteration(cache.at("twicerepeated").data()).get(), IterationCount({})); + ASSERT_EQ(decode_iteration(cache.at("twicerepeated").data()).get(), IterationCount({0})); } ASSERT_THROW(communicator_.pre_receive(), PortClosed); @@ -277,3 +328,89 @@ TEST_P(libmuscle_repeater_communicator, repeater_filters_discard_messages) { INSTANTIATE_TEST_SUITE_P( repeated, libmuscle_repeater_communicator, ::testing::Values("repeat", "pad")); + + +TEST_F(libmuscle_reducer_communicator, reducer_filters) { + mock_receive_messages({{"component.init", {I({0}), I({1}), M({})}}}); + + auto & deposit = communicator_.server_.deposit; + + auto cache = communicator_.pre_receive(); + ASSERT_EQ(decode_iteration(cache.at("init").data()).get(), IterationCount({0})); + // Send some messages on O_I + for (std::size_t i = 0; i < 5; ++i) { + communicator_.send_message("out", Message(i, Data(i), Settings())); + ASSERT_FALSE(deposit.called()); + } + // Send on O_F + communicator_.send_message("final", Message(5, Data("data"), Settings())); + ASSERT_EQ(deposit.call_args_list.size(), 2); + ASSERT_EQ(std::get<0>(deposit.call_args_list[0]), "parent.in"); + ASSERT_EQ(std::get<0>(deposit.call_args_list[1]), "sibling.in2"); + deposit.call_args_list.clear(); + + // Pre-receive will send cached LAST message to sibling.in + cache = communicator_.pre_receive(); + ASSERT_EQ(decode_iteration(cache.at("init").data()).get(), IterationCount({1})); + // N.B. we don't send the [1] milestone to sibling.in due to the LAST filter, only + // the cached message + ASSERT_EQ(deposit.call_args_list.size(), 1); + ASSERT_EQ(std::get<0>(deposit.call_args_list[0]), "sibling.in"); + auto sent_message = std::get<1>(deposit.call_args_list[0]); + ASSERT_EQ(sent_message->timestamp, 4.0); + deposit.call_args_list.clear(); + + // Skip O_I and send on O_F + communicator_.send_message("final", Message(10, Data("data"), Settings())); + ASSERT_EQ(deposit.call_args_list.size(), 2); + ASSERT_EQ(std::get<0>(deposit.call_args_list[0]), "parent.in"); + ASSERT_EQ(std::get<0>(deposit.call_args_list[1]), "sibling.in2"); + deposit.call_args_list.clear(); + + // Pre-receive will first send cached LAST message to sibling.in, then receive + // Milestone([]) and trigger: + // - Cached LAST message on "final" to aunt.init + // - Cached LAST LAST message on "out" to uncle.init + // - Milestone([]) to sibling.in, sibling.in2, parent.in + ASSERT_THROW(communicator_.pre_receive(), PortClosed); + ASSERT_EQ(deposit.call_args_list.size(), 6.0); + + std::unordered_map>> messages_per_peer_port; + for (auto & call : deposit.call_args_list) { + std::string peer_port(std::get<0>(call)); + std::shared_ptr message(std::get<1>(call)); + auto it = messages_per_peer_port.find(peer_port); + if (it == messages_per_peer_port.end()) { + // Not found + messages_per_peer_port.emplace( + peer_port, std::vector>({message})); + } else { + it->second.emplace_back(message); + } + } + + // O_I -> last -> sibling.in + auto & messages = messages_per_peer_port.at("sibling.in"); + ASSERT_EQ(messages.size(), 2); + // No message was sent on O_I this reuse loop, so LAST generates an empty message + ASSERT_EQ(messages[0]->timestamp, -std::numeric_limits::infinity()); + ASSERT_TRUE(messages[0]->data.is_nil()); + ASSERT_TRUE(is_milestone(messages[1]->data)); + ASSERT_TRUE(Milestone(messages[1]->data).is_final_milestone()); + + // Just milestones + for (auto & peer_port : {"sibling.in2", "parent.in"}) { + messages = messages_per_peer_port.at(peer_port); + ASSERT_EQ(messages.size(), 1); + ASSERT_TRUE(is_milestone(messages[0]->data)); + ASSERT_TRUE(Milestone(messages[0]->data).is_final_milestone()); + } + + // O_I -> last last -> uncle.init + ASSERT_EQ(messages_per_peer_port.at("uncle.init").size(), 1); + ASSERT_EQ(messages_per_peer_port.at("uncle.init")[0]->timestamp, 4.0); + + // O_F -> last -> aunt.init + ASSERT_EQ(messages_per_peer_port.at("aunt.init").size(), 1); + ASSERT_EQ(messages_per_peer_port.at("aunt.init")[0]->timestamp, 10.0); +} diff --git a/src/cpp/libmuscle/tests/test_timeline_manager.cpp b/src/cpp/libmuscle/tests/test_timeline_manager.cpp index 65418495..a435b266 100644 --- a/src/cpp/libmuscle/tests/test_timeline_manager.cpp +++ b/src/cpp/libmuscle/tests/test_timeline_manager.cpp @@ -119,9 +119,9 @@ struct libmuscle_timeline_manager : ::testing::Test { void create( PortsDescription const & declared_ports, std::unordered_map const & timelines = {}, - bool include_settings = false) { + bool include_settings = false, Timeline timeline = Timeline(":")) { port_manager_ = make_port_manager(declared_ports, timelines, {}, include_settings); - tm_ = std::make_unique(*port_manager_); + tm_ = std::make_unique(*port_manager_, timeline); ASSERT_FALSE(tm_->start_reuse_iteration().is_set()); } }; @@ -146,7 +146,8 @@ struct libmuscle_full_timeline_manager : libmuscle_timeline_manager { {"out_a2_2", Timeline(":A2")}, {"in_a2", Timeline(":A2")}, }, - true); + true, + Timeline(":a")); } }; @@ -155,7 +156,7 @@ struct libmuscle_vector_timeline_manager : libmuscle_timeline_manager { libmuscle_vector_timeline_manager() { port_manager_ = make_port_manager( PortsDescription{{Operator::O_F, {"out_v[]"}}}, {}, {{"out_v", {3}}}); - tm_ = std::make_unique(*port_manager_); + tm_ = std::make_unique(*port_manager_, Timeline(":")); tm_->start_reuse_iteration(); EXPECT_EQ(tm_->record_pre_received_iteration_counts({}), IterationCount()); } @@ -195,6 +196,11 @@ TEST_F(libmuscle_full_timeline_manager, record_received_message_f_init_raises_on ASSERT_THROW(tm_->record_pre_received_iteration_counts({{3}, {4}}), std::logic_error); } +TEST_F(libmuscle_full_timeline_manager, check_pre_receive_counts_match_timeline) { + ASSERT_THROW(tm_->record_pre_received_iteration_counts({{}}), std::runtime_error); + ASSERT_THROW(tm_->record_pre_received_iteration_counts({{1, 2}}), std::runtime_error); +} + TEST_F(libmuscle_timeline_manager, o_f_can_send_immediately_when_no_f_init_connections) { create(PortsDescription{{Operator::O_F, {"out_f"}}}); tm_->record_pre_received_iteration_counts({}); @@ -340,7 +346,7 @@ TEST_F(libmuscle_full_timeline_manager, get_state_and_restore_state_round_trip) TimelineState state = tm_->get_state(); - TimelineManager restored(*port_manager_); + TimelineManager restored(*port_manager_, Timeline(":a")); restored.restore_state(state); TimelineState restored_state = restored.get_state(); diff --git a/src/cpp/libmuscle/timeline_manager.cpp b/src/cpp/libmuscle/timeline_manager.cpp index 5e7694ec..f5d53a25 100644 --- a/src/cpp/libmuscle/timeline_manager.cpp +++ b/src/cpp/libmuscle/timeline_manager.cpp @@ -438,8 +438,9 @@ void SubTimelineManager::missing_actions(ExpectedActions & result) const { // when a test mocks TimelineManager but still includes this file for them. #ifndef LIBMUSCLE_MOCK_TIMELINE_MANAGER -TimelineManager::TimelineManager(PortManager const & port_manager) - : port_manager_(port_manager) +TimelineManager::TimelineManager(PortManager const & port_manager, Timeline const & timeline) + : timeline_(timeline) + , port_manager_(port_manager) , receive_(port_manager.get_connected_ports(Operator::F_INIT, Optional())) , send_(port_manager.get_connected_ports(Operator::O_F, Optional())) , submanagers_() @@ -472,6 +473,11 @@ IterationCount TimelineManager::record_pre_received_iteration_counts( throw std::runtime_error( "Internal error: received F_INIT iteration count " + to_string(new_iteration) + " is not newer than the previous iteration " + to_string(iteration_.get())); + if (new_iteration.size() != timeline_.size()) + throw std::runtime_error( + "Received unexpected F_INIT iteration count: " + to_string(new_iteration) + + ". Was expecting an iteration count with " + std::to_string(timeline_.size()) + + " elements, since we are in timeline " + std::string(timeline_)); iteration_ = new_iteration; return new_iteration; } diff --git a/src/cpp/libmuscle/timeline_manager.hpp b/src/cpp/libmuscle/timeline_manager.hpp index d930938a..8a01faf7 100644 --- a/src/cpp/libmuscle/timeline_manager.hpp +++ b/src/cpp/libmuscle/timeline_manager.hpp @@ -290,7 +290,8 @@ class TimelineManager { * @param port_manager The (already connected) port manager for this * instance. */ - explicit TimelineManager(PortManager const & port_manager); + explicit TimelineManager( + PortManager const & port_manager, ::ymmsl::Timeline const & timeline); /** Check and update the timeline state before sending on the given * port. @@ -371,6 +372,7 @@ class TimelineManager { private: IterationCount check_send_o_f_(Port const & port, Optional slot); + ::ymmsl::Timeline timeline_; PortManager const & port_manager_; TimelinePorts receive_; TimelinePorts send_; diff --git a/src/cpp/ymmsl/ports.cpp b/src/cpp/ymmsl/ports.cpp index de21ca39..3c1f9dc4 100644 --- a/src/cpp/ymmsl/ports.cpp +++ b/src/cpp/ymmsl/ports.cpp @@ -1,5 +1,7 @@ #include +#include + ::std::size_t (::std::hash<::ymmsl::impl::Timeline>::operator())( argument_type const & timeline) const noexcept @@ -21,6 +23,14 @@ bool Timeline::operator==(Timeline const & rhs) const { return timeline_ == rhs.timeline_; } +std::size_t Timeline::size() const { + if (timeline_.empty() || timeline_ == ":") return 0; + std::size_t num_colons = std::count(timeline_.begin(), timeline_.end(), ':'); + if (timeline_[0] != ':') + return num_colons + 1; + return num_colons; +} + Port::Port(Identifier const & name, Operator oper, Timeline const & timeline) : name(name) , oper(oper) diff --git a/src/cpp/ymmsl/ports.hpp b/src/cpp/ymmsl/ports.hpp index b7ed2742..dcc7a142 100644 --- a/src/cpp/ymmsl/ports.hpp +++ b/src/cpp/ymmsl/ports.hpp @@ -55,6 +55,10 @@ class Timeline { */ bool operator==(Timeline const & rhs) const; + /** Return the number of parts in the Timeline. + */ + std::size_t size() const; + private: std::string timeline_; }; diff --git a/src/cpp/ymmsl/tests/test_ports.cpp b/src/cpp/ymmsl/tests/test_ports.cpp index be052974..29ae6784 100644 --- a/src/cpp/ymmsl/tests/test_ports.cpp +++ b/src/cpp/ymmsl/tests/test_ports.cpp @@ -26,6 +26,15 @@ TEST(ymmsl_timeline, hash_consistent_with_equality) { ASSERT_EQ(hasher(Timeline(":A1")), hasher(Timeline(":A1"))); } +TEST(ymmsl_timeline, size) { + ASSERT_EQ(Timeline("").size(), 0); + ASSERT_EQ(Timeline(":").size(), 0); + ASSERT_EQ(Timeline("a").size(), 1); + ASSERT_EQ(Timeline(":a").size(), 1); + ASSERT_EQ(Timeline("a:b").size(), 2); + ASSERT_EQ(Timeline(":a:b").size(), 2); +} + TEST(ymmsl_port, test_port) { auto ep1 = Port(Identifier("test_in"), Operator::F_INIT); diff --git a/src/python/libmuscle/communicator.py b/src/python/libmuscle/communicator.py index f1a3b721..4c42aa1a 100644 --- a/src/python/libmuscle/communicator.py +++ b/src/python/libmuscle/communicator.py @@ -155,6 +155,21 @@ def __init__( self._message_cache: MPPCacheType = {} """Message cache for pre-received messages.""" + self._outgoing_timeline_length: dict[Reference, int] = {} + """Size of IterationCount, after applying the reducer filters, per peer port. + + Keys are references to peer ports: ``component + port``. The outgoing timeline + length is the size of the IterationCount after applying the reducer filters and + determines in which (parent) timeline these messages are sent. + + If our timeline is ":macro:micro" then: + - outgoing_timeline_length = 0: send on the root (":") timeline + - outgoing_timeline_length = 1: send on the ":macro" timeline + - outgoing_timeline_length = 2: send on the ":macro:micro" timeline + """ + self._reducer_cache: dict[Reference, MPPMessage] = {} + """Message cache for reducer filters.""" + def get_locations(self) -> list[str]: """Returns a list of locations that we can be reached at. @@ -177,8 +192,9 @@ def set_peer_info(self, peer_info: PeerInfo) -> None: Args: peer_info: Information about the peers. """ + self._timeline = self._manager.get_timeline() self._peer_info = peer_info - self._timeline_manager = TimelineManager(self._port_manager) + self._timeline_manager = TimelineManager(self._port_manager, self._timeline) self._prepare_conduit_filters() def set_receive_timeout(self, receive_timeout: float) -> None: @@ -277,6 +293,12 @@ def send_message( message.data, iteration, ) + peer_port = recv_endpoint.kernel + recv_endpoint.port + if peer_port in self._outgoing_timeline_length: + result = self._apply_reduce_filters(peer_port, mpp_message) + if result is None: + continue + mpp_message = result encoded_message = mpp_message.encoded() profile_event.message_size = len(memoryview(encoded_message)) self._server.deposit(recv_endpoint.ref(), encoded_message) @@ -290,6 +312,65 @@ def send_message( elif message.data.is_final_milestone(): port.set_closed(slot) + def _apply_reduce_filters( + self, peer_port: Reference, message: MPPMessage + ) -> Optional[MPPMessage]: + """Apply reduce filters to a message sent on a conduit with reduce filters. + + User-provided messages (through instance.send()) will be stored (overwriting any + existing message). For milestones this method decides if the milestone should be + sent, or a cached message, or nothing at all. + + Args: + peer_port: Peer port (component + port) to send to. + message: MPPMessage to be checked. + + Returns: + The MPPMessage to send, or None if we do not need to send anything. + """ + message.message_number = -1 # GH#411: Disabled checkpointing for reducer filter + + reduced_count = self._outgoing_timeline_length[peer_port] + + if not isinstance(message.data, Milestone): + # Reduce the message iteration count to match with the timeline we send to + message.iteration = message.iteration[:reduced_count] + self._reducer_cache[message.receiver] = message + _logger.debug("Message for %s stored in cache", message.receiver) + return None + + # Decide whether to send the milestone, ignore it, or send a cached message. + n_milestone = len(message.data.iteration) + if n_milestone < reduced_count: + # Milestone from ancestor timeline: send it + return message + + elif n_milestone == reduced_count: + # This is the target timeline after reduce filters applied: we need to + # send the cached message (or make up an empty one) and discard the + # milestone: + cached_msg = self._reducer_cache.pop(message.receiver, None) + if cached_msg is None: + _logger.info( + "No cached message available to send because this instance did " + "not run. Sending an empty message to %s instead.", + message.receiver, + ) + message.data = None + return message + + assert cached_msg.iteration == message.iteration + _logger.debug("Sending cached message to %s", message.receiver) + return cached_msg + + else: + _logger.debug( + "Ignored %s for %s because of LAST filters.", + message.data, + message.receiver, + ) + return None + def pre_receive(self) -> FInitCacheType: """Pre-receive on all connected F_INIT ports and S ports with repeat filters. @@ -316,8 +397,6 @@ def pre_receive(self) -> FInitCacheType: if message.iteration != milestone_iteration } - # TODO: send buffered message for reducer filters - if milestone.is_final_milestone(): raise PortClosed() @@ -530,7 +609,10 @@ def _receive_message(self, port_name: str, slot: Optional[int]) -> MPPMessage: self._profiler.record_event(receive_event) expected_message_number = port.get_num_messages(slot) - if expected_message_number != mpp_message.message_number: + if ( + mpp_message.message_number >= 0 # GH#411: negative for reducer filters + and expected_message_number != mpp_message.message_number + ): if ( expected_message_number - 1 == mpp_message.message_number and port.is_resuming(slot) @@ -686,12 +768,11 @@ def _prepare_conduit_filters(self) -> None: """Check which ports are connected with a conduit filter and initialize the associated logic. """ + peer_info = self._peer_info # Repeater filters for operator in (Operator.F_INIT, Operator.S): for port in self._port_manager.get_connected_ports(operator): - filters = self._peer_info.get_filters_for_receiver( - self._kernel + port.name - ) + filters = peer_info.get_filters_for_receiver(self._kernel + port.name) # Only keep the repeater filters, the sending component handles reducers filters = [filter for filter in filters if filter.is_repeater()] if filters: @@ -699,7 +780,17 @@ def _prepare_conduit_filters(self) -> None: if operator is Operator.F_INIT or filters: self._pre_receive_ports.append(port) - # TODO: reducer filters + # Reducer filters + for operator in (Operator.O_I, Operator.O_F): + for port in self._port_manager.get_connected_ports(operator): + for peer_port in peer_info.get_peer_ports(port.name): + filters = peer_info.get_filters_for_receiver(peer_port) + # Count the reducer filters, receiving component handles repeaters + n_reducers = sum(1 for filter in filters if filter.is_reducer()) + if n_reducers > 0: + self._outgoing_timeline_length[peer_port] = ( + len(self._timeline) + len(port.timeline) - n_reducers + ) def _pad_message( self, cur_iteration: IterationCount, filters: list[ConduitFilter] diff --git a/src/python/libmuscle/manager/manager.py b/src/python/libmuscle/manager/manager.py index db81bc7d..daca4d02 100644 --- a/src/python/libmuscle/manager/manager.py +++ b/src/python/libmuscle/manager/manager.py @@ -45,6 +45,16 @@ def __init__( configuration: The simulation configuration. run_dir: Main working directory. """ + # TEMP: check for checkpoints combined with reducer filters: + if configuration.checkpoints and any( + any(filter.is_reducer() for filter in conduit.filters) + for conduit in configuration.root_model().conduits + ): + raise NotImplementedError( + "This version of MUSCLE3 does not support checkpoints when using " + "reducer filters. See https://github.com/multiscale/muscle3/issues/411" + ) + self._configuration = configuration self._run_dir = run_dir log_dir = self._run_dir.path if self._run_dir else Path.cwd() diff --git a/src/python/libmuscle/mpp_message.py b/src/python/libmuscle/mpp_message.py index e6593859..61bf7637 100644 --- a/src/python/libmuscle/mpp_message.py +++ b/src/python/libmuscle/mpp_message.py @@ -206,7 +206,7 @@ def __init__( self.message_number = message_number self.iteration = iteration if isinstance(data, np.ndarray): - self.data = Grid(data) + self.data: Any = Grid(data) else: self.data = data diff --git a/src/python/libmuscle/test/test_communicator.py b/src/python/libmuscle/test/test_communicator.py index 9a5eef93..c7cd67fe 100644 --- a/src/python/libmuscle/test/test_communicator.py +++ b/src/python/libmuscle/test/test_communicator.py @@ -112,15 +112,13 @@ def test_create_communicator(communicator, mpp_server): pass -def test_set_peer_info_creates_timeline_manager( - communicator, connected_port_manager, timeline_manager -): +def test_set_peer_info_creates_timeline_manager(communicator, timeline_manager): peer_info = MagicMock() communicator.set_peer_info(peer_info) assert communicator._peer_info == peer_info - timeline_manager.assert_called_once_with(connected_port_manager) + timeline_manager.assert_called_once() assert communicator._timeline_manager == timeline_manager.return_value diff --git a/src/python/libmuscle/test/test_communicator_conduit_filters.py b/src/python/libmuscle/test/test_communicator_conduit_filters.py index 0a1ae0be..8f5b5b6c 100644 --- a/src/python/libmuscle/test/test_communicator_conduit_filters.py +++ b/src/python/libmuscle/test/test_communicator_conduit_filters.py @@ -1,12 +1,12 @@ from typing import Union -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, MagicMock, call, patch import pytest -from ymmsl.v0_2 import Conduit, ConduitFilter, Operator, Port, Settings +from ymmsl.v0_2 import Conduit, ConduitFilter, Operator, Port, Settings, Timeline from ymmsl.v0_2 import Identifier as Id from ymmsl.v0_2 import Reference as Ref -from libmuscle.communicator import Communicator, PortClosed +from libmuscle.communicator import Communicator, Message, PortClosed from libmuscle.mpp_message import Milestone, MPPMessage from libmuscle.peer_info import PeerInfo from libmuscle.port_manager import PortManager @@ -19,6 +19,12 @@ def mpp_client(): yield MPPClient.return_value +@pytest.fixture +def mpp_server(): + with patch("libmuscle.communicator.MPPServer") as MPPServer: + yield MPPServer.return_value + + @pytest.fixture(params=["repeat", "pad"]) def repeat_filter(request): return request.param @@ -27,8 +33,10 @@ def repeat_filter(request): @pytest.fixture() def repeater_communicator(repeat_filter, mpp_client): port_manager = PortManager([], None) + mock_manager = MagicMock() + mock_manager.get_timeline.return_value = Timeline(":parent3:parent2:parent1") communicator = Communicator( - Ref("component"), [], port_manager, MagicMock(), MagicMock() + Ref("component"), [], port_manager, MagicMock(), mock_manager ) peer_info = PeerInfo( Ref("component"), @@ -63,6 +71,89 @@ def repeater_communicator(repeat_filter, mpp_client): communicator.shutdown() +@pytest.fixture +def reducer_communicator(mpp_client, mpp_server): + port_manager = PortManager([], None) + mock_manager = MagicMock() + mock_manager.get_timeline.return_value = Timeline(":parent") + communicator = Communicator( + Ref("component"), [], port_manager, MagicMock(), mock_manager + ) + peer_info = PeerInfo( + Ref("component"), + [], + [ + Conduit("parent.out", "component.init"), + Conduit("component.final", "parent.in"), + Conduit("component.final", "sibling.in2"), + # Reducer filter on O_I port + Conduit("component.out", "sibling.in", "last"), + # Reducer filter on O_F port + Conduit("component.final", "aunt.init", "last"), + # Double reducer filter on O_I port + Conduit("component.out", "uncle.init", "last last"), + ], + {Ref("parent"): [], Ref("aunt"): [], Ref("uncle"): [], Ref("sibling"): []}, + {Ref("parent"): [], Ref("aunt"): [], Ref("uncle"): [], Ref("sibling"): []}, + [ + Port(Id("init"), Operator.F_INIT), + Port(Id("out"), Operator.O_I, Timeline("component")), + Port(Id("final"), Operator.O_F), + ], + ) + port_manager.connect_ports(peer_info) + communicator.set_peer_info(peer_info) + assert communicator._outgoing_timeline_length == { + "sibling.in": 1, + "aunt.init": 0, + "uncle.init": 0, + } + yield communicator + communicator.shutdown() + + +@pytest.fixture +def repeater_reducer_communicator(repeat_filter, mpp_client, mpp_server): + conduit = Conduit("component.out", "sibling.in", "last " + repeat_filter) + + port_manager = PortManager([], None) + mock_manager = MagicMock() + mock_manager.get_timeline.return_value = Timeline(":") + component = Communicator( + Ref("component"), [], port_manager, MagicMock(), mock_manager + ) + peer_info = PeerInfo( + Ref("component"), + [], + [conduit], + {Ref("sibling"): []}, + {Ref("sibling"): []}, + [Port(Id("out"), Operator.O_I, Timeline("component"))], + ) + port_manager.connect_ports(peer_info) + component.set_peer_info(peer_info) + + sibling_port_manager = PortManager([], None) + sibling = Communicator( + Ref("sibling"), [], sibling_port_manager, MagicMock(), MagicMock() + ) + sibling_peer_info = PeerInfo( + Ref("sibling"), + [], + [conduit], + {Ref("component"): []}, + {Ref("component"): []}, + [Port(Id("in"), Operator.S)], + ) + sibling_port_manager.connect_ports(sibling_peer_info) + sibling.set_peer_info(sibling_peer_info) + assert sibling._repeat_filters == {"in": [ConduitFilter(repeat_filter)]} + + component.pre_receive() + yield component, sibling + component.shutdown() + + def mock_receive_messages( mpp_client, data: dict[str, list[Union[IterationCount, Milestone]]] ): @@ -105,16 +196,17 @@ def side_effect(peer, _): def test_repeater_filters(repeater_communicator, mpp_client, repeat_filter): - twicerepeated_messages = [[], Milestone([])] - repeated_messages = [[0], [1], [2], Milestone([])] + twicerepeated_messages = [[0], Milestone([])] + repeated_messages = [[0, 0], [0, 1], [0, 2], Milestone([0]), Milestone([])] unfiltered_messages = [ - [0, 0], - [0, 1], - Milestone([0]), + [0, 0, 0], + [0, 0, 1], + Milestone([0, 0]), # parent is allowed to send 0 messages on its O_I port in an iteration - Milestone([1]), - [2, 0], - Milestone([2]), + Milestone([0, 1]), + [0, 2, 0], + Milestone([0, 2]), + Milestone([0]), Milestone([]), ] mock_receive_messages( @@ -130,19 +222,19 @@ def test_repeater_filters(repeater_communicator, mpp_client, repeat_filter): is_padded = repeat_filter == "pad" cache = repeater_communicator.pre_receive() - assert cache[("unfiltered", None)].data == [0, 0] - assert cache[("repeated", None)].data == [0] - assert cache[("twicerepeated", None)].data == [] + assert cache[("unfiltered", None)].data == [0, 0, 0] + assert cache[("repeated", None)].data == [0, 0] + assert cache[("twicerepeated", None)].data == [0] cache = repeater_communicator.pre_receive() - assert cache[("unfiltered", None)].data == [0, 1] - assert cache[("repeated", None)].data == (None if is_padded else [0]) - assert cache[("twicerepeated", None)].data == (None if is_padded else []) + assert cache[("unfiltered", None)].data == [0, 0, 1] + assert cache[("repeated", None)].data == (None if is_padded else [0, 0]) + assert cache[("twicerepeated", None)].data == (None if is_padded else [0]) cache = repeater_communicator.pre_receive() - assert cache[("unfiltered", None)].data == [2, 0] - assert cache[("repeated", None)].data == [2] - assert cache[("twicerepeated", None)].data == (None if is_padded else []) + assert cache[("unfiltered", None)].data == [0, 2, 0] + assert cache[("repeated", None)].data == [0, 2] + assert cache[("twicerepeated", None)].data == (None if is_padded else [0]) with pytest.raises(PortClosed): repeater_communicator.pre_receive() @@ -251,3 +343,100 @@ def test_repeater_filters_no_finit(mpp_client, repeat_filter): # Cleanup communicator.shutdown() + + +def test_reducer_filters(reducer_communicator, mpp_client, mpp_server): + mock_receive_messages(mpp_client, {"component.init": [[0], [1], Milestone([])]}) + + cache = reducer_communicator.pre_receive() + assert cache[("init", None)].data == [0] + # Send some messages on O_I + for i in range(5): + reducer_communicator.send_message("out", Message(i, data="data")) + mpp_server.deposit.assert_not_called() + # Send on O_F + reducer_communicator.send_message("final", Message(5, data="data")) + assert mpp_server.deposit.call_args_list == [ + call("parent.in", ANY), + call("sibling.in2", ANY), + ] + mpp_server.deposit.reset_mock() + + # Pre-receive will send cached LAST message to sibling.in + cache = reducer_communicator.pre_receive() + assert cache[("init", None)].data == [1] + # N.B. we don't send the [1] milestone to sibling.in due to the LAST filter, only + # the cached message + mpp_server.deposit.assert_called_once_with("sibling.in", ANY) + sent_message = MPPMessage.from_bytes(mpp_server.deposit.call_args.args[1]) + assert sent_message.timestamp == 4 # The last message on O_I + mpp_server.deposit.reset_mock() + + # Skip O_I and send on O_F + reducer_communicator.send_message("final", Message(10, data="data")) + assert mpp_server.deposit.call_args_list == [ + call("parent.in", ANY), + call("sibling.in2", ANY), + ] + mpp_server.deposit.reset_mock() + + # Pre-receive will first send cached LAST message to sibling.in, then receive + # Milestone([]) and trigger: + # - Cached LAST message on "final" to aunt.init + # - Cached LAST LAST message on "out" to uncle.init + # - Milestone([]) to sibling.in, sibling.in2, parent.in + with pytest.raises(PortClosed): + reducer_communicator.pre_receive() + assert mpp_server.deposit.call_count == 6 + + messages_per_peer_port = {} + for item in mpp_server.deposit.call_args_list: + msg = MPPMessage.from_bytes(item.args[1]) + messages_per_peer_port.setdefault(item.args[0], []).append(msg) + + # O_I -> last -> sibling.in + assert len(messages_per_peer_port["sibling.in"]) == 2 + # No message was sent on O_I this reuse loop, so LAST generates an empty message: + assert messages_per_peer_port["sibling.in"][0].timestamp == float("-inf") + assert messages_per_peer_port["sibling.in"][0].data is None + assert isinstance(messages_per_peer_port["sibling.in"][1].data, Milestone) + assert messages_per_peer_port["sibling.in"][1].data.is_final_milestone() + + # Just milestones + for peer_port in ["sibling.in2", "parent.in"]: + assert len(messages_per_peer_port[peer_port]) == 1 + assert isinstance(messages_per_peer_port[peer_port][0].data, Milestone) + assert messages_per_peer_port[peer_port][0].data.is_final_milestone() + + # O_I -> last last -> uncle.init + assert len(messages_per_peer_port["uncle.init"]) == 1 + assert messages_per_peer_port["uncle.init"][0].timestamp == 4 + + # O_F -> last -> aunt.init + assert len(messages_per_peer_port["aunt.init"]) == 1 + assert messages_per_peer_port["aunt.init"][0].timestamp == 10 + + +def test_combined_reducer_and_repeater_filters( + repeater_reducer_communicator, mpp_client, mpp_server, repeat_filter +): + component, sibling = repeater_reducer_communicator + + for i in range(3): + component.send_message("out", Message(i, data=f"value_{i}")) + mpp_server.deposit.assert_not_called() # messages are cached, not yet forwarded + + # Closing broadcasts the closing milestone, which releases the cached last message. + component._close_outgoing_ports() + assert mpp_server.deposit.call_count == 1 + peer, encoded = mpp_server.deposit.call_args.args + assert peer == "sibling.in" + + # Feed the bytes "component" put on the wire into "sibling"'s receive. + mpp_client.receive.return_value = (encoded, MagicMock()) + sibling.pre_receive() + + is_padded = repeat_filter == "pad" + for i in range(3): + msg = sibling.receive_s_message("in") + assert msg.data == (None if i and is_padded else "value_2") diff --git a/src/python/libmuscle/test/test_timeline_manager.py b/src/python/libmuscle/test/test_timeline_manager.py index ee5410cb..88617eb4 100644 --- a/src/python/libmuscle/test/test_timeline_manager.py +++ b/src/python/libmuscle/test/test_timeline_manager.py @@ -31,7 +31,14 @@ def include_settings(request: pytest.FixtureRequest) -> bool: @pytest.fixture -def timeline_manager(has_f_init: bool, include_settings: bool) -> TimelineManager: +def timeline(request: pytest.FixtureRequest) -> Timeline: + return getattr(request, "param", Timeline(":")) + + +@pytest.fixture +def timeline_manager( + has_f_init: bool, include_settings: bool, timeline: Timeline +) -> TimelineManager: conduits = [ Conduit("component.out_f", "peer_f.in"), Conduit("component.out_a1", "peer_a1.in"), @@ -69,20 +76,20 @@ def timeline_manager(has_f_init: bool, include_settings: bool) -> TimelineManage peer_info = PeerInfo(Ref("component"), [], conduits, peer_dims, {}, ymmsl_ports) pm.connect_ports(peer_info) - tm = TimelineManager(pm) + tm = TimelineManager(pm, timeline) assert tm.start_reuse_iteration() is None return tm @pytest.fixture -def vector_timeline_manager() -> TimelineManager: +def vector_timeline_manager(timeline: Timeline) -> TimelineManager: declared_ports = {Operator.O_F: ["out_v[]"]} pm = PortManager([], declared_ports) conduits = [Conduit("component.out_v", "peer.in")] peer_info = PeerInfo(Ref("component"), [], conduits, {Ref("peer"): [3]}, {}, []) pm.connect_ports(peer_info) - tm = TimelineManager(pm) + tm = TimelineManager(pm, timeline) assert tm.start_reuse_iteration() is None assert tm.record_pre_received_iteration_counts([]) == [] return tm @@ -274,6 +281,7 @@ def test_check_send_message_o_i_blocked_when_o_i_leads_and_incomplete( ) +@pytest.mark.parametrize("timeline", [Timeline(":a:b:c")], indirect=True) def test_check_pre_receive_increments(timeline_manager: TimelineManager) -> None: assert timeline_manager.record_pre_received_iteration_counts( [[1, 2, 3], [1, 2], [1], [], [1, 2, 3], [1, 2]] @@ -288,6 +296,14 @@ def test_check_pre_receive_increments(timeline_manager: TimelineManager) -> None ) == [1, 2, 4] +def test_check_pre_receive_counts_match_timeline( + timeline_manager: TimelineManager, +) -> None: + with pytest.raises(RuntimeError, match="iteration count with 0 elements"): + timeline_manager.record_pre_received_iteration_counts([[1]]) + + +@pytest.mark.parametrize("timeline", [Timeline(":a")], indirect=True) def test_check_pre_receive_iterations_when_iteration_differs( timeline_manager: TimelineManager, ) -> None: @@ -391,6 +407,7 @@ def test_check_receive_message_s_blocked_when_s_leads_and_incomplete( ) +@pytest.mark.parametrize("timeline", [Timeline(":a")], indirect=True) def test_finish_reuse_iteration_resets_when_complete( timeline_manager: TimelineManager, ) -> None: @@ -415,6 +432,7 @@ def test_finish_reuse_iteration_resets_when_complete( ) +@pytest.mark.parametrize("timeline", [Timeline(":a")], indirect=True) def test_finish_reuse_iteration_raises_when_incomplete( timeline_manager: TimelineManager, ) -> None: @@ -436,6 +454,7 @@ def test_finish_reuse_iteration_raises_when_incomplete( ) +@pytest.mark.parametrize("timeline", [Timeline(":a")], indirect=True) def test_get_state_and_restore_state_round_trip( timeline_manager: TimelineManager, ) -> None: @@ -449,7 +468,7 @@ def test_get_state_and_restore_state_round_trip( # Restore into a fresh TimelineManager, as would happen after loading a # snapshot in a new process, from an independent but identically # configured PortManager. - restored = TimelineManager(timeline_manager._port_manager) + restored = TimelineManager(timeline_manager._port_manager, Timeline(":")) restored.restore_state(timeline_state) assert restored.get_state() == timeline_state diff --git a/src/python/libmuscle/timeline_manager.py b/src/python/libmuscle/timeline_manager.py index f3eb0b03..1d65deda 100644 --- a/src/python/libmuscle/timeline_manager.py +++ b/src/python/libmuscle/timeline_manager.py @@ -252,7 +252,7 @@ class TimelineManager: ports that receive (F_INIT or S). """ - def __init__(self, port_manager: PortManager) -> None: + def __init__(self, port_manager: PortManager, timeline: Timeline) -> None: """Create a TimelineManager. The port_manager must already have its ports connected to their peers, since @@ -263,6 +263,7 @@ def __init__(self, port_manager: PortManager) -> None: Args: port_manager: The (already connected) port manager for this instance. """ + self._timeline = timeline self._port_manager = port_manager self._send = TimelinePorts(port_manager.get_connected_ports(Operator.O_F)) subtimelines = port_manager.list_subtimelines() @@ -320,6 +321,12 @@ def record_pre_received_iteration_counts( f"Internal error: received F_INIT iteration count {new_iteration} " f"is not newer than the previous iteration {self._iteration}." ) + if len(new_iteration) != len(self._timeline): + raise RuntimeError( + f"Received unexpected F_INIT iteration count: {new_iteration}. Was " + f"expecting an iteration count with {len(self._timeline)} elements, " + f"since we are in timeline {self._timeline}" + ) self._iteration = new_iteration return self._iteration