From 9352abdff5f5181c4703700b028415a2464d863e Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 7 Apr 2026 13:29:13 +0300 Subject: [PATCH 01/22] 246 update evaluation docs --- docs/source/evaluation.rst | 124 ++++++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 7 deletions(-) diff --git a/docs/source/evaluation.rst b/docs/source/evaluation.rst index 7259a6ca..862f242b 100644 --- a/docs/source/evaluation.rst +++ b/docs/source/evaluation.rst @@ -1,13 +1,123 @@ Evaluation ========== -The evaluation of a conversational recommender system (CRS) is performed by first generating dialogues between the CRS and the user simulator, then computing evaluation measures on these synthetic dialogues. -The evaluation scripts are located in the directory `scripts/evaluation`. +UserSimCRS evaluates conversational recommender systems (CRSs) on exported dialogues. The evaluation pipeline loads dialogues from a JSON file, computes one or more metrics, and stores the results as JSON together with the resolved configuration. -Currently, we provide the following evaluation scripts: +A default evaluation configuration is provided in `config/default/config_evaluation.yaml`. - * **Dialogue quality evaluation**: Evaluates the dialogue quality with regards to five aspects: recommendation relevance, communication style, fluency, conversational flow, and overall satisfaction. The scores for each aspect are obtained from a large language model (LLM) hosted on a Ollama server. - * **Satisfaction evaluation**: Evaluates the user satisfaction using a pre-trained model from DialogueKit. - * **Utility evaluation**: Evaluates dialogues based on user-centric utility metrics: success rate, successful recommendation round ratio, and reward-per-dialogue-length. -Please refer to the documentation of each script for more details on how to run them. \ No newline at end of file +Usage +----- + +Run evaluation with: + +.. code-block:: shell + + python -m usersimcrs.run_evaluation -c + + +Some parameters can also be overridden from the command line, for example: + +.. code-block:: shell + + python -m usersimcrs.run_evaluation \ + -c config/default/config_evaluation.yaml \ + --dialogues data/datasets/moviebot/annotated_dialogues.json \ + --metrics satisfaction success_rate \ + --output data/evaluation/results.json + + +Configuration +------------- + +The evaluation configuration is defined in a YAML file. The main parameters are: + + * `dialogues`: Path to the dialogues JSON file. + * `metrics`: List of metrics to compute. + * `output`: Path to the JSON file where evaluation results will be saved. + * `quality_aspects`: Quality aspects to evaluate when `quality` is included in `metrics`. + * `quality_llm_interface`: LLM interface configuration used by the quality metric. + * `user_nlu_config`: Configuration file used to instantiate the user-side NLU for utility metrics. + * `agent_nlu_config`: Configuration file used to instantiate the agent-side NLU for utility metrics. + * `recommendation_intent_labels`: Intent labels that mark recommendation turns. + * `accept_intent_labels`: Intent labels that mark acceptance. + * `reject_intent_labels`: Intent labels that mark rejection. + + +The following metrics are currently supported: + + * `quality` + * `satisfaction` + * `success_rate` + * `successful_recommendation_round_ratio` + * `reward_per_dialogue_length` + + +Metric Overview +--------------- + +Quality +""""""" + +The quality metric uses an LLM to score each dialogue aspect separately. The supported aspects are defined by ``QualityRubrics``: + + * `REC_RELEVANCE` + * `COM_STYLE` + * `FLUENCY` + * `CONV_FLOW` + * `OVERALL_SAT` + + +When `quality` is requested, the configuration must include `quality_llm_interface`. + + +Satisfaction +"""""""""""" + +The satisfaction metric uses the pre-trained DialogueKit satisfaction classifier and returns one score per dialogue. + + +Utility Metrics +""""""""""""""" + +The utility metrics are: + + * **Success rate**: Returns `1.0` if at least one recommendation was accepted in the dialogue, otherwise `0.0`. + * **Successful recommendation round ratio**: Returns the ratio of accepted recommendation rounds to all recommendation rounds in the dialogue. + * **Reward per dialogue length**: Returns the number of accepted recommendations divided by the total number of utterances in the dialogue. + + +If the input dialogues are not already annotated, UserSimCRS annotates them in place using the NLU components loaded from `user_nlu_config` and `agent_nlu_config`. + +When any utility metric is requested, the following configuration fields are required: + + * `user_nlu_config` + * `agent_nlu_config` + * `recommendation_intent_labels` + * `accept_intent_labels` + * `reject_intent_labels` + + +Output +------ + +The evaluation script writes two files: + + * The JSON result file specified by `output`. + * A companion metadata file with the suffix `.meta.yaml`, containing the resolved configuration. + + +The result JSON contains: + + * `dialogues_path`: Path to the evaluated dialogues. + * `metrics_requested`: List of requested metrics. + * `metrics`: Metric results. + + +For `satisfaction` and all utility metrics, each metric entry contains: + + * `per_dialogue`: Mapping from conversation ID to score. + * `summary_by_agent`: Aggregate statistics per agent (`count`, `min`, `max`, `mean`, `stdev`). + + +For `quality`, the output is grouped by aspect. Each aspect contains its own `per_dialogue` scores and `summary_by_agent` statistics. From 6ce1a4cdd201f6af48eb0ff0e99dce3fa7f9288e Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 21 Apr 2026 16:21:33 +0200 Subject: [PATCH 02/22] change docs --- docs/source/evaluation.rst | 55 ++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/docs/source/evaluation.rst b/docs/source/evaluation.rst index 862f242b..1aea9d7e 100644 --- a/docs/source/evaluation.rst +++ b/docs/source/evaluation.rst @@ -1,7 +1,7 @@ Evaluation ========== -UserSimCRS evaluates conversational recommender systems (CRSs) on exported dialogues. The evaluation pipeline loads dialogues from a JSON file, computes one or more metrics, and stores the results as JSON together with the resolved configuration. +UserSimCRS evaluates conversational recommender systems (CRSs) on previously generated synthetic dialogues. The evaluation pipeline loads dialogues from a JSON file, computes one or more metrics, and stores the results as JSON together with the copy of the configuration used. A default evaluation configuration is provided in `config/default/config_evaluation.yaml`. @@ -21,10 +21,12 @@ Some parameters can also be overridden from the command line, for example: .. code-block:: shell python -m usersimcrs.run_evaluation \ - -c config/default/config_evaluation.yaml \ + -c \ --dialogues data/datasets/moviebot/annotated_dialogues.json \ --metrics satisfaction success_rate \ - --output data/evaluation/results.json + --output-dir data/evaluation + +Run ``python -m usersimcrs.run_evaluation -h`` for the full list of available command-line arguments. The configuration fields used by these arguments are described below. Configuration @@ -34,11 +36,10 @@ The evaluation configuration is defined in a YAML file. The main parameters are: * `dialogues`: Path to the dialogues JSON file. * `metrics`: List of metrics to compute. - * `output`: Path to the JSON file where evaluation results will be saved. + * `output_dir`: Directory where evaluation results and metadata will be saved. * `quality_aspects`: Quality aspects to evaluate when `quality` is included in `metrics`. * `quality_llm_interface`: LLM interface configuration used by the quality metric. - * `user_nlu_config`: Configuration file used to instantiate the user-side NLU for utility metrics. - * `agent_nlu_config`: Configuration file used to instantiate the agent-side NLU for utility metrics. + * `annotate_dialogues`: Whether dialogues should be annotated before metric computation. * `recommendation_intent_labels`: Intent labels that mark recommendation turns. * `accept_intent_labels`: Intent labels that mark acceptance. * `reject_intent_labels`: Intent labels that mark rejection. @@ -59,6 +60,8 @@ Metric Overview Quality """"""" +:py:class:`usersimcrs.evaluation.quality_metric.QualityMetric` + The quality metric uses an LLM to score each dialogue aspect separately. The supported aspects are defined by ``QualityRubrics``: * `REC_RELEVANCE` @@ -74,37 +77,61 @@ When `quality` is requested, the configuration must include `quality_llm_interfa Satisfaction """""""""""" +:py:class:`usersimcrs.evaluation.satisfaction_metric.SatisfactionMetric` + The satisfaction metric uses the pre-trained DialogueKit satisfaction classifier and returns one score per dialogue. Utility Metrics """"""""""""""" -The utility metrics are: +The utility metrics capture recommendation outcomes from annotated dialogues. If the input dialogues are not already annotated, they can be annotated before evaluation by enabling `annotate_dialogues` and providing `user_nlu` and `agent_nlu` configurations. For additional context on their role in the evaluation setup, see `Bernard and Balog, 2026 `_. + + +Success Rate +'''''''''''' + +:py:class:`usersimcrs.evaluation.success_rate_metric.SuccessRateMetric` + +Returns `1.0` if at least one recommendation was accepted in the dialogue, otherwise `0.0`. + - * **Success rate**: Returns `1.0` if at least one recommendation was accepted in the dialogue, otherwise `0.0`. - * **Successful recommendation round ratio**: Returns the ratio of accepted recommendation rounds to all recommendation rounds in the dialogue. - * **Reward per dialogue length**: Returns the number of accepted recommendations divided by the total number of utterances in the dialogue. +Successful Recommendation Round Ratio +'''''''''''''''''''''''''''''''''''''' +:py:class:`usersimcrs.evaluation.successful_recommendation_round_ratio_metric.SuccessfulRecommendationRoundRatioMetric` + +Returns the ratio of accepted recommendation rounds to all recommendation rounds in the dialogue. + + +Reward per Dialogue Length +'''''''''''''''''''''''''' + +:py:class:`usersimcrs.evaluation.reward_per_dialogue_length_metric.RewardPerDialogueLengthMetric` + +Returns the number of accepted recommendations divided by the total number of utterances in the dialogue. If the input dialogues are not already annotated, UserSimCRS annotates them in place using the NLU components loaded from `user_nlu_config` and `agent_nlu_config`. When any utility metric is requested, the following configuration fields are required: - * `user_nlu_config` - * `agent_nlu_config` * `recommendation_intent_labels` * `accept_intent_labels` * `reject_intent_labels` +When `annotate_dialogues` is enabled, the following configuration fields are also required: + + * `user_nlu` + * `agent_nlu` + Output ------ The evaluation script writes two files: - * The JSON result file specified by `output`. - * A companion metadata file with the suffix `.meta.yaml`, containing the resolved configuration. + * `results.json` in the directory specified by `output_dir`. + * `config_evaluation.meta.yaml` in the same directory, containing a copy of the configuration used. The result JSON contains: From e78f37d80ce7d18340c6d0bfd0cb7259badf58b4 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 16 Jun 2026 09:43:13 +0200 Subject: [PATCH 03/22] update the doc, add examples --- docs/source/evaluation.rst | 77 ++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/docs/source/evaluation.rst b/docs/source/evaluation.rst index 1aea9d7e..aaed3401 100644 --- a/docs/source/evaluation.rst +++ b/docs/source/evaluation.rst @@ -64,11 +64,11 @@ Quality The quality metric uses an LLM to score each dialogue aspect separately. The supported aspects are defined by ``QualityRubrics``: - * `REC_RELEVANCE` - * `COM_STYLE` - * `FLUENCY` - * `CONV_FLOW` - * `OVERALL_SAT` + * `REC_RELEVANCE`: Recommendation relevance measures how closely the recommended items align with the user’s preferences and needs. + * `COM_STYLE`: Communication style corresponds to the conciseness and clarity of the responses. + * `FLUENCY`: Fluency is the degree of naturalness of the responses compared to human-generated responses. + * `CONV_FLOW`: Conversational flow assesses the coherence and consistency of the conversation. + * `OVERALL_SAT`: Overall satisfaction encapsulates the user’s holistic experience. When `quality` is requested, the configuration must include `quality_llm_interface`. @@ -82,10 +82,10 @@ Satisfaction The satisfaction metric uses the pre-trained DialogueKit satisfaction classifier and returns one score per dialogue. -Utility Metrics -""""""""""""""" +User Utility Metrics +"""""""""""""""""""" -The utility metrics capture recommendation outcomes from annotated dialogues. If the input dialogues are not already annotated, they can be annotated before evaluation by enabling `annotate_dialogues` and providing `user_nlu` and `agent_nlu` configurations. For additional context on their role in the evaluation setup, see `Bernard and Balog, 2026 `_. +The user utility metrics capture recommendation outcomes from annotated dialogues. If the input dialogues are not already annotated, they can be annotated before evaluation by enabling `annotate_dialogues` and providing `user_nlu` and `agent_nlu` configurations. For additional context on their role in the evaluation setup, see `Bernard and Balog, 2025 `_. Success Rate @@ -111,9 +111,7 @@ Reward per Dialogue Length Returns the number of accepted recommendations divided by the total number of utterances in the dialogue. -If the input dialogues are not already annotated, UserSimCRS annotates them in place using the NLU components loaded from `user_nlu_config` and `agent_nlu_config`. - -When any utility metric is requested, the following configuration fields are required: +When any user utility metric is requested, the following configuration fields are required: * `recommendation_intent_labels` * `accept_intent_labels` @@ -141,10 +139,65 @@ The result JSON contains: * `metrics`: Metric results. -For `satisfaction` and all utility metrics, each metric entry contains: +For `satisfaction` and all user utility metrics, each metric entry contains: * `per_dialogue`: Mapping from conversation ID to score. * `summary_by_agent`: Aggregate statistics per agent (`count`, `min`, `max`, `mean`, `stdev`). For `quality`, the output is grouped by aspect. Each aspect contains its own `per_dialogue` scores and `summary_by_agent` statistics. + +Example output structure: + +.. code-block:: json + + { + "dialogues_path": "data/datasets/moviebot/annotated_dialogues.json", + "metrics_requested": ["satisfaction", "success_rate", "quality"], + "metrics": { + "satisfaction": { + "per_dialogue": { + "conv_001": 0.82 + }, + "summary_by_agent": { + "moviebot": { + "count": 1, + "min": 0.82, + "max": 0.82, + "mean": 0.82, + "stdev": 0.0 + } + } + }, + "success_rate": { + "per_dialogue": { + "conv_001": 1.0 + }, + "summary_by_agent": { + "moviebot": { + "count": 1, + "min": 1.0, + "max": 1.0, + "mean": 1.0, + "stdev": 0.0 + } + } + }, + "quality": { + "REC_RELEVANCE": { + "per_dialogue": { + "conv_001": 4.5 + }, + "summary_by_agent": { + "moviebot": { + "count": 1, + "min": 4.5, + "max": 4.5, + "mean": 4.5, + "stdev": 0.0 + } + } + } + } + } + } From 73ab49da82f8c415905027cbd1fa4581214a2605 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 16 Jun 2026 13:55:50 +0200 Subject: [PATCH 04/22] add dynamic information need --- tests/core/test_information_need.py | 41 ++++++++++-- usersimcrs/core/information_need.py | 65 ++++++++++++++++++- .../agenda_based/interaction_model.py | 32 +++++++++ .../llm/llm_dual_prompt_user_simulator.py | 1 + .../llm/llm_single_prompt_user_simulator.py | 2 + usersimcrs/simulator/user_simulator.py | 37 +++++++++++ 6 files changed, 169 insertions(+), 9 deletions(-) diff --git a/tests/core/test_information_need.py b/tests/core/test_information_need.py index a339292e..a15caf87 100644 --- a/tests/core/test_information_need.py +++ b/tests/core/test_information_need.py @@ -59,16 +59,27 @@ def test_get_requestable_slots(information_need: InformationNeed) -> None: information_need: Information need. """ assert information_need.get_requestable_slots() == ["PLOT", "RATING"] - information_need.requested_slots["RATING"] = 4.5 + information_need.mark_request_complete("RATING", 4.5) assert information_need.get_requestable_slots() == ["PLOT"] -def test_to_dict(information_need: InformationNeed) -> None: - """Test to_dict. - - Args: - information_need: Information need. - """ +def test_to_dict() -> None: + """Test to_dict.""" + information_need = InformationNeed( + [ + Item( + "1", + { + "GENRE": "Comedy", + "DIRECTOR": "Steven Spielberg", + "RATING": 4.5, + "PLOT": "A movie plot", + }, + ) + ], + {"GENRE": "Comedy", "DIRECTOR": "Steven Spielberg"}, + ["PLOT", "RATING"], + ) assert information_need.to_dict() == { "target_items": [ { @@ -83,6 +94,14 @@ def test_to_dict(information_need: InformationNeed) -> None: ], "constraints": {"GENRE": "Comedy", "DIRECTOR": "Steven Spielberg"}, "requests": ["PLOT", "RATING"], + "constraint_states": { + "GENRE": InformationNeed.INCOMPLETE, + "DIRECTOR": InformationNeed.INCOMPLETE, + }, + "request_states": { + "PLOT": InformationNeed.INCOMPLETE, + "RATING": InformationNeed.INCOMPLETE, + }, } @@ -140,3 +159,11 @@ def test_from_dict() -> None: loaded_information_need.requested_slots == expected_information_need.requested_slots ) + assert ( + loaded_information_need.constraint_states + == expected_information_need.constraint_states + ) + assert ( + loaded_information_need.request_states + == expected_information_need.request_states + ) diff --git a/usersimcrs/core/information_need.py b/usersimcrs/core/information_need.py index 11636522..bb35d429 100644 --- a/usersimcrs/core/information_need.py +++ b/usersimcrs/core/information_need.py @@ -11,7 +11,7 @@ import random from collections import defaultdict -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item import Item @@ -54,11 +54,17 @@ def generate_random_information_need( class InformationNeed: + INCOMPLETE = "incomplete" + ATTEMPTED = "attempted" + COMPLETE = "complete" + def __init__( self, target_items: List[Item], constraints: Dict[str, Any], requests: List[str], + constraint_states: Optional[Dict[str, str]] = None, + request_states: Optional[Dict[str, str]] = None, ) -> None: """Initializes an information need. @@ -67,12 +73,23 @@ def __init__( constraints: Slot-value pairs representing constraints on the item of interest. requests: Slots representing the desired information. + constraint_states: Optional states for constraints. + request_states: Optional states for requests. """ self.target_items = target_items self.constraints = constraints self.requested_slots = defaultdict( None, {slot: None for slot in requests} ) + self.constraint_states = { + slot: (constraint_states or {}).get(slot, self.INCOMPLETE) + for slot in constraints + } + + self.request_states = { + slot: (request_states or {}).get(slot, self.INCOMPLETE) + for slot in requests + } def get_constraint_value(self, slot: str) -> Any: """Returns the value of a constraint slot. @@ -90,9 +107,49 @@ def get_requestable_slots(self) -> List[str]: return [ slot for slot in self.requested_slots - if not self.requested_slots[slot] + if self.request_states.get(slot) != self.COMPLETE + ] + + def _mark_state( + self, states: Dict[str, str], slot: str, state: str + ) -> None: + if slot in states: + states[slot] = state + + def mark_constraint_attempted(self, slot: str) -> None: + self._mark_state(self.constraint_states, slot, self.ATTEMPTED) + + def mark_constraint_complete(self, slot: str) -> None: + self._mark_state(self.constraint_states, slot, self.COMPLETE) + + def mark_request_attempted(self, slot: str) -> None: + self._mark_state(self.request_states, slot, self.ATTEMPTED) + + def mark_request_complete(self, slot: str, value: Any = None) -> None: + if slot in self.request_states: + self.requested_slots[slot] = value + self.request_states[slot] = self.COMPLETE + + def get_goal_progress(self) -> Dict[str, int]: + states = [ + *self.constraint_states.values(), + *self.request_states.values(), ] + return { + self.INCOMPLETE: states.count(self.INCOMPLETE), + self.ATTEMPTED: states.count(self.ATTEMPTED), + self.COMPLETE: states.count(self.COMPLETE), + "total": len(states), + } + + def get_goal_completion_ratio(self) -> float: + """Returns the fraction of completed goal components.""" + progress = self.get_goal_progress() + if progress["total"] == 0: + return 1.0 + return progress[self.COMPLETE] / progress["total"] + @classmethod def from_dict(cls, data: Dict[str, Any]) -> InformationNeed: """Creates information need from a dictionary.""" @@ -101,6 +158,8 @@ def from_dict(cls, data: Dict[str, Any]) -> InformationNeed: target_items=target_items, constraints=data["constraints"], requests=data["requests"], + constraint_states=data.get("constraint_states"), + request_states=data.get("request_states"), ) def to_dict(self) -> Dict[str, Any]: @@ -112,4 +171,6 @@ def to_dict(self) -> Dict[str, Any]: ], "constraints": self.constraints, "requests": list(self.requested_slots.keys()), + "constraint_states": dict(self.constraint_states), + "request_states": dict(self.request_states), } diff --git a/usersimcrs/simulator/agenda_based/interaction_model.py b/usersimcrs/simulator/agenda_based/interaction_model.py index 97318b90..c75f5b44 100644 --- a/usersimcrs/simulator/agenda_based/interaction_model.py +++ b/usersimcrs/simulator/agenda_based/interaction_model.py @@ -315,6 +315,9 @@ def update_agenda( """ current_state = self.dialogue_state_tracker.get_current_state() agent_dialogue_acts = current_state.agent_dialogue_acts[-1] + self._update_goal_state_from_agent_dialogue_acts( + information_need, agent_dialogue_acts + ) user_dialogue_acts = [] for dialogue_act in agent_dialogue_acts: if self.is_agent_intent_elicit(dialogue_act.intent): @@ -355,6 +358,25 @@ def update_agenda( self.agenda.push_dialogue_acts(user_dialogue_acts) self.agenda.clean_agenda(information_need) + def _update_goal_state_from_agent_dialogue_acts( + self, + information_need: InformationNeed, + agent_dialogue_acts: List[DialogueAct], + ) -> None: + """Updates goal progress based on the latest agent dialogue acts.""" + for dialogue_act in agent_dialogue_acts: + for annotation in dialogue_act.annotations: + slot = annotation.slot + value = annotation.value + if slot in information_need.request_states: + if value is not None: + information_need.mark_request_complete(slot, value) + else: + information_need.mark_request_attempted(slot) + + if slot in information_need.constraint_states: + information_need.mark_constraint_attempted(slot) + def _get_preference_intent( self, preference: float, preference_model: PreferenceModel ) -> Intent: @@ -424,6 +446,8 @@ def _generate_elicit_response_dialogue_acts( [SlotValueAnnotation(elicited_slot, elicited_value)], ) ) + if elicited_slot in information_need.constraint_states: + information_need.mark_constraint_attempted(elicited_slot) else: # Agent is asking about value preferences on a given slot, e.g., # "What movie genre would you prefer?" The value is taken either @@ -451,6 +475,8 @@ def _generate_elicit_response_dialogue_acts( annotations, ) ) + if elicited_slot in information_need.constraint_states: + information_need.mark_constraint_complete(elicited_slot) else: user_dialogue_acts.append(DialogueAct(self.INTENT_DONT_KNOW)) # type: ignore[attr-defined] # noqa @@ -528,6 +554,7 @@ def _generate_inquire_response_dialogue_acts( slot = slot_value_annotation.slot if slot_value_annotation.value is None: if slot in information_need.get_requestable_slots(): + information_need.mark_request_attempted(slot) user_dialogue_acts.append( DialogueAct( self.INTENT_YES, [SlotValueAnnotation(slot)] # type: ignore[attr-defined] # noqa @@ -548,6 +575,8 @@ def _generate_inquire_response_dialogue_acts( slot = random.choice(requestable_slots) else: slot = random.choice(self._domain.get_requestable_slots()) + if slot in information_need.request_states: + information_need.mark_request_attempted(slot) user_dialogue_acts.append( DialogueAct(self.INQUIRE, [SlotValueAnnotation(slot)]) # type: ignore[attr-defined] # noqa ) @@ -594,6 +623,7 @@ def _sample_next_user_dialogue_acts( slot, value = random.choice( list(information_need.constraints.items()) ) + information_need.mark_constraint_complete(slot) user_dialogue_acts.append( DialogueAct( @@ -604,6 +634,8 @@ def _sample_next_user_dialogue_acts( slot = random.choice(information_need.get_requestable_slots()) if not slot: slot = random.choice(self._domain.get_requestable_slots()) + if slot in information_need.request_states: + information_need.mark_request_attempted(slot) user_dialogue_acts.append( DialogueAct(sampled_intent, [SlotValueAnnotation(slot)]) ) diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index 4036a57d..7601141f 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -64,6 +64,7 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ + self._update_goal_state_from_agent_text(agent_utterance) self.generation_prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index 3d413242..cf9f3e1b 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -53,10 +53,12 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ + self._update_goal_state_from_agent_text(agent_utterance) self.prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) user_utterance = self.llm_interface.generate_utterance(self.prompt) + self._update_goal_state_from_user_text(user_utterance) self.prompt.update_prompt_context( user_utterance, DialogueParticipant.USER ) diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index af238c4e..7b900360 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -1,6 +1,7 @@ """User simulator abstract class.""" from abc import ABC, abstractmethod +import re from dialoguekit.core.annotated_utterance import AnnotatedUtterance from dialoguekit.core.utterance import Utterance @@ -29,6 +30,42 @@ def get_new_information_need(self) -> None: self._domain, self._item_collection ) + def _normalize_text(self, text: str) -> str: + """Normalizes text for simple slot/value matching.""" + return " ".join(re.sub(r"[_-]", " ", text.lower()).split()) + + def _update_goal_state_from_agent_text(self, utterance: Utterance) -> None: + """Updates goal state from a raw agent utterance.""" + raw = getattr(utterance, "text", "") + text = self._normalize_text(raw) + + if not text: + return + + is_question = "?" in raw + + for slot in self.information_need.request_states: + normalized_slot = self._normalize_text(slot) + + if normalized_slot not in text: + continue + + if is_question: + self.information_need.mark_request_attempted(slot) + else: + self.information_need.mark_request_complete(slot, raw.strip()) + + for slot, value in self.information_need.constraints.items(): + normalized_slot = self._normalize_text(slot) + values = value if isinstance(value, list) else [value] + + normalized_values = [self._normalize_text(str(v)) for v in values] + + if normalized_slot in text or any( + v in text for v in normalized_values + ): + self.information_need.mark_constraint_attempted(slot) + @abstractmethod def _generate_response(self, agent_utterance: Utterance) -> Utterance: """Generates response to the agent utterance. From cb6a1f988ec84944bca4643526e0e6664f3aa629 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 23 Jun 2026 13:51:20 +0200 Subject: [PATCH 05/22] return evaluation doc --- docs/source/evaluation.rst | 204 ++----------------------------------- 1 file changed, 7 insertions(+), 197 deletions(-) diff --git a/docs/source/evaluation.rst b/docs/source/evaluation.rst index aaed3401..7259a6ca 100644 --- a/docs/source/evaluation.rst +++ b/docs/source/evaluation.rst @@ -1,203 +1,13 @@ Evaluation ========== -UserSimCRS evaluates conversational recommender systems (CRSs) on previously generated synthetic dialogues. The evaluation pipeline loads dialogues from a JSON file, computes one or more metrics, and stores the results as JSON together with the copy of the configuration used. +The evaluation of a conversational recommender system (CRS) is performed by first generating dialogues between the CRS and the user simulator, then computing evaluation measures on these synthetic dialogues. +The evaluation scripts are located in the directory `scripts/evaluation`. -A default evaluation configuration is provided in `config/default/config_evaluation.yaml`. +Currently, we provide the following evaluation scripts: + * **Dialogue quality evaluation**: Evaluates the dialogue quality with regards to five aspects: recommendation relevance, communication style, fluency, conversational flow, and overall satisfaction. The scores for each aspect are obtained from a large language model (LLM) hosted on a Ollama server. + * **Satisfaction evaluation**: Evaluates the user satisfaction using a pre-trained model from DialogueKit. + * **Utility evaluation**: Evaluates dialogues based on user-centric utility metrics: success rate, successful recommendation round ratio, and reward-per-dialogue-length. -Usage ------ - -Run evaluation with: - -.. code-block:: shell - - python -m usersimcrs.run_evaluation -c - - -Some parameters can also be overridden from the command line, for example: - -.. code-block:: shell - - python -m usersimcrs.run_evaluation \ - -c \ - --dialogues data/datasets/moviebot/annotated_dialogues.json \ - --metrics satisfaction success_rate \ - --output-dir data/evaluation - -Run ``python -m usersimcrs.run_evaluation -h`` for the full list of available command-line arguments. The configuration fields used by these arguments are described below. - - -Configuration -------------- - -The evaluation configuration is defined in a YAML file. The main parameters are: - - * `dialogues`: Path to the dialogues JSON file. - * `metrics`: List of metrics to compute. - * `output_dir`: Directory where evaluation results and metadata will be saved. - * `quality_aspects`: Quality aspects to evaluate when `quality` is included in `metrics`. - * `quality_llm_interface`: LLM interface configuration used by the quality metric. - * `annotate_dialogues`: Whether dialogues should be annotated before metric computation. - * `recommendation_intent_labels`: Intent labels that mark recommendation turns. - * `accept_intent_labels`: Intent labels that mark acceptance. - * `reject_intent_labels`: Intent labels that mark rejection. - - -The following metrics are currently supported: - - * `quality` - * `satisfaction` - * `success_rate` - * `successful_recommendation_round_ratio` - * `reward_per_dialogue_length` - - -Metric Overview ---------------- - -Quality -""""""" - -:py:class:`usersimcrs.evaluation.quality_metric.QualityMetric` - -The quality metric uses an LLM to score each dialogue aspect separately. The supported aspects are defined by ``QualityRubrics``: - - * `REC_RELEVANCE`: Recommendation relevance measures how closely the recommended items align with the user’s preferences and needs. - * `COM_STYLE`: Communication style corresponds to the conciseness and clarity of the responses. - * `FLUENCY`: Fluency is the degree of naturalness of the responses compared to human-generated responses. - * `CONV_FLOW`: Conversational flow assesses the coherence and consistency of the conversation. - * `OVERALL_SAT`: Overall satisfaction encapsulates the user’s holistic experience. - - -When `quality` is requested, the configuration must include `quality_llm_interface`. - - -Satisfaction -"""""""""""" - -:py:class:`usersimcrs.evaluation.satisfaction_metric.SatisfactionMetric` - -The satisfaction metric uses the pre-trained DialogueKit satisfaction classifier and returns one score per dialogue. - - -User Utility Metrics -"""""""""""""""""""" - -The user utility metrics capture recommendation outcomes from annotated dialogues. If the input dialogues are not already annotated, they can be annotated before evaluation by enabling `annotate_dialogues` and providing `user_nlu` and `agent_nlu` configurations. For additional context on their role in the evaluation setup, see `Bernard and Balog, 2025 `_. - - -Success Rate -'''''''''''' - -:py:class:`usersimcrs.evaluation.success_rate_metric.SuccessRateMetric` - -Returns `1.0` if at least one recommendation was accepted in the dialogue, otherwise `0.0`. - - -Successful Recommendation Round Ratio -'''''''''''''''''''''''''''''''''''''' - -:py:class:`usersimcrs.evaluation.successful_recommendation_round_ratio_metric.SuccessfulRecommendationRoundRatioMetric` - -Returns the ratio of accepted recommendation rounds to all recommendation rounds in the dialogue. - - -Reward per Dialogue Length -'''''''''''''''''''''''''' - -:py:class:`usersimcrs.evaluation.reward_per_dialogue_length_metric.RewardPerDialogueLengthMetric` - -Returns the number of accepted recommendations divided by the total number of utterances in the dialogue. - -When any user utility metric is requested, the following configuration fields are required: - - * `recommendation_intent_labels` - * `accept_intent_labels` - * `reject_intent_labels` - -When `annotate_dialogues` is enabled, the following configuration fields are also required: - - * `user_nlu` - * `agent_nlu` - - -Output ------- - -The evaluation script writes two files: - - * `results.json` in the directory specified by `output_dir`. - * `config_evaluation.meta.yaml` in the same directory, containing a copy of the configuration used. - - -The result JSON contains: - - * `dialogues_path`: Path to the evaluated dialogues. - * `metrics_requested`: List of requested metrics. - * `metrics`: Metric results. - - -For `satisfaction` and all user utility metrics, each metric entry contains: - - * `per_dialogue`: Mapping from conversation ID to score. - * `summary_by_agent`: Aggregate statistics per agent (`count`, `min`, `max`, `mean`, `stdev`). - - -For `quality`, the output is grouped by aspect. Each aspect contains its own `per_dialogue` scores and `summary_by_agent` statistics. - -Example output structure: - -.. code-block:: json - - { - "dialogues_path": "data/datasets/moviebot/annotated_dialogues.json", - "metrics_requested": ["satisfaction", "success_rate", "quality"], - "metrics": { - "satisfaction": { - "per_dialogue": { - "conv_001": 0.82 - }, - "summary_by_agent": { - "moviebot": { - "count": 1, - "min": 0.82, - "max": 0.82, - "mean": 0.82, - "stdev": 0.0 - } - } - }, - "success_rate": { - "per_dialogue": { - "conv_001": 1.0 - }, - "summary_by_agent": { - "moviebot": { - "count": 1, - "min": 1.0, - "max": 1.0, - "mean": 1.0, - "stdev": 0.0 - } - } - }, - "quality": { - "REC_RELEVANCE": { - "per_dialogue": { - "conv_001": 4.5 - }, - "summary_by_agent": { - "moviebot": { - "count": 1, - "min": 4.5, - "max": 4.5, - "mean": 4.5, - "stdev": 0.0 - } - } - } - } - } - } +Please refer to the documentation of each script for more details on how to run them. \ No newline at end of file From ba8a58e96263a9f780f2bc58f76cc599b3974d71 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 23 Jun 2026 14:45:20 +0200 Subject: [PATCH 06/22] add new preference generation function --- usersimcrs/core/information_need.py | 92 +++++++++++++++++++ .../agenda_based/interaction_model.py | 14 ++- usersimcrs/simulator/user_simulator.py | 12 ++- 3 files changed, 116 insertions(+), 2 deletions(-) diff --git a/usersimcrs/core/information_need.py b/usersimcrs/core/information_need.py index bb35d429..e7d981b1 100644 --- a/usersimcrs/core/information_need.py +++ b/usersimcrs/core/information_need.py @@ -16,6 +16,9 @@ from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item import Item from usersimcrs.items.item_collection import ItemCollection +from dialoguekit.core.slot_value_annotation import SlotValueAnnotation + +from usersimcrs.user_modeling.preference_model import PreferenceModel def generate_random_information_need( @@ -53,6 +56,85 @@ def generate_random_information_need( return InformationNeed([target_item], constraints, requests) +def generate_preference_grounded_information_need( + domain: SimulationDomain, + item_collection: ItemCollection, + preference_model: PreferenceModel, + max_constraints: int = 2, +) -> InformationNeed: + """Generates an information need aligned with a preference model. + + The function first samples positive slot preferences from the preference + model, then tries to find an item matching them. If no suitable item is + found, it falls back to a random item while preserving any discovered + preference-aligned constraints. + + Args: + domain: Domain knowledge. + item_collection: Collection of items. + preference_model: Preference model of the same simulated user. + max_constraints: Maximum number of preference-grounded constraints. + + Returns: + Information need. + """ + preferred_constraints: Dict[str, Any] = {} + for slot in domain.get_informable_slots(): + value, score = preference_model.get_slot_preference(slot) + if value is None or score < preference_model.PREFERENCE_THRESHOLD: + continue + preferred_constraints[slot] = value + + if preferred_constraints: + sampled_slots = random.sample( + list(preferred_constraints.keys()), + min(max_constraints, len(preferred_constraints)), + ) + constraints = { + slot: preferred_constraints[slot] for slot in sampled_slots + } + else: + constraints = {} + + matching_items = item_collection.get_items_by_properties( + [ + SlotValueAnnotation(slot, value) + for slot, value in constraints.items() + ] + ) + + if matching_items: + target_item = max( + matching_items, + key=lambda item: preference_model.get_item_preference(item.id), + ) + else: + target_item = item_collection.get_random_item() + if not constraints: + informable_slots = set(domain.get_informable_slots()).intersection( + target_item.properties.keys() + ) + if informable_slots: + num_constraints = random.randint(1, len(informable_slots)) + for slot in random.sample( + list(informable_slots), num_constraints + ): + constraints[slot] = target_item.get_property(slot) + + requestable_slots = [ + slot + for slot in domain.get_requestable_slots() + if slot not in constraints + ] + if requestable_slots: + num_requests = random.randint(1, len(requestable_slots)) + requests = random.sample(requestable_slots, num_requests) + else: + requests = [] + + return InformationNeed([target_item], constraints, requests) + + class InformationNeed: INCOMPLETE = "incomplete" ATTEMPTED = "attempted" @@ -113,6 +195,16 @@ def get_requestable_slots(self) -> List[str]: def _mark_state( self, states: Dict[str, str], slot: str, state: str ) -> None: + """Updates the state of a tracked slot. + + Args: + states: Mapping from slots to their current states. + slot: Slot whose state should be updated. + state: New state to assign to the slot. + + Returns: + None. + """ if slot in states: states[slot] = state diff --git a/usersimcrs/simulator/agenda_based/interaction_model.py b/usersimcrs/simulator/agenda_based/interaction_model.py index c75f5b44..800d011f 100644 --- a/usersimcrs/simulator/agenda_based/interaction_model.py +++ b/usersimcrs/simulator/agenda_based/interaction_model.py @@ -363,7 +363,19 @@ def _update_goal_state_from_agent_dialogue_acts( information_need: InformationNeed, agent_dialogue_acts: List[DialogueAct], ) -> None: - """Updates goal progress based on the latest agent dialogue acts.""" + """Updates goal progress based on the latest agent dialogue acts. + + Marks requested slots as attempted or complete depending on whether + the agent provided a value, and marks mentioned constraint slots as + attempted. + + Args: + information_need: Information need to update. + agent_dialogue_acts: Latest dialogue acts produced by the agent. + + Returns: + None. + """ for dialogue_act in agent_dialogue_acts: for annotation in dialogue_act.annotations: slot = annotation.slot diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index 7b900360..05b6f501 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -31,7 +31,17 @@ def get_new_information_need(self) -> None: ) def _normalize_text(self, text: str) -> str: - """Normalizes text for simple slot/value matching.""" + """Normalizes text for simple slot and value matching. + + Lowercases the text, replaces underscores and hyphens with spaces, + and collapses repeated whitespace. + + Args: + text: Text to normalize. + + Returns: + Normalized text string. + """ return " ".join(re.sub(r"[_-]", " ", text.lower()).split()) def _update_goal_state_from_agent_text(self, utterance: Utterance) -> None: From 4c273f1c3a27da09ac05118f529a0bcd66146ded Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 30 Jun 2026 13:59:17 +0200 Subject: [PATCH 07/22] refactor information need --- tests/core/test_information_need.py | 8 +- usersimcrs/core/information_need.py | 148 +++++++++++------- .../agenda_based/interaction_model.py | 28 +++- .../llm/llm_dual_prompt_user_simulator.py | 2 +- .../llm/llm_single_prompt_user_simulator.py | 2 +- usersimcrs/simulator/user_simulator.py | 78 ++++++++- 6 files changed, 185 insertions(+), 81 deletions(-) diff --git a/tests/core/test_information_need.py b/tests/core/test_information_need.py index a15caf87..5ab7c62d 100644 --- a/tests/core/test_information_need.py +++ b/tests/core/test_information_need.py @@ -95,12 +95,12 @@ def test_to_dict() -> None: "constraints": {"GENRE": "Comedy", "DIRECTOR": "Steven Spielberg"}, "requests": ["PLOT", "RATING"], "constraint_states": { - "GENRE": InformationNeed.INCOMPLETE, - "DIRECTOR": InformationNeed.INCOMPLETE, + "GENRE": InformationNeed.SLOT_STATE_INCOMPLETE, + "DIRECTOR": InformationNeed.SLOT_STATE_INCOMPLETE, }, "request_states": { - "PLOT": InformationNeed.INCOMPLETE, - "RATING": InformationNeed.INCOMPLETE, + "PLOT": InformationNeed.SLOT_STATE_INCOMPLETE, + "RATING": InformationNeed.SLOT_STATE_INCOMPLETE, }, } diff --git a/usersimcrs/core/information_need.py b/usersimcrs/core/information_need.py index e7d981b1..c168eaa3 100644 --- a/usersimcrs/core/information_need.py +++ b/usersimcrs/core/information_need.py @@ -60,66 +60,69 @@ def generate_preference_grounded_information_need( domain: SimulationDomain, item_collection: ItemCollection, preference_model: PreferenceModel, - max_constraints: int = 2, ) -> InformationNeed: """Generates an information need aligned with a preference model. - The function first samples positive slot preferences from the preference - model, then tries to find an item matching them. If no suitable item is - found, it falls back to a random item while preserving any discovered - preference-aligned constraints. + The function first identifies a target item aligned with the preference + model, then samples matching properties from that item as constraints. + This keeps the constraints grounded in a real item while still biasing + the information need toward the user's preferences. Args: domain: Domain knowledge. item_collection: Collection of items. - preference_model: Preference model of the same simulated user. - max_constraints: Maximum number of preference-grounded constraints. + preference_model: Preference model of the simulated user. Returns: Information need. """ - preferred_constraints: Dict[str, Any] = {} + preferred_constraints: Dict[str, str] = {} for slot in domain.get_informable_slots(): value, score = preference_model.get_slot_preference(slot) if value is None or score < preference_model.PREFERENCE_THRESHOLD: continue preferred_constraints[slot] = value + target_item = item_collection.get_random_item() if preferred_constraints: - sampled_slots = random.sample( - list(preferred_constraints.keys()), - min(max_constraints, len(preferred_constraints)), + anchor_slot = random.choice(list(preferred_constraints.keys())) + anchor_value = preferred_constraints[anchor_slot] + matching_items = item_collection.get_items_by_properties( + [SlotValueAnnotation(anchor_slot, anchor_value)] ) + liked_items = [ + item + for item in matching_items + if preference_model.get_item_preference(item.id) + >= preference_model.PREFERENCE_THRESHOLD + ] + + if liked_items: + target_item = random.choice(liked_items) + elif matching_items: + target_item = random.choice(matching_items) + + preference_aligned_slots = [ + slot + for slot, value in preferred_constraints.items() + if target_item.get_property(slot) == value + ] + + if preference_aligned_slots: + num_constraints = random.randint(1, len(preference_aligned_slots)) + sampled_slots = random.sample(preference_aligned_slots, num_constraints) constraints = { - slot: preferred_constraints[slot] for slot in sampled_slots + slot: target_item.get_property(slot) for slot in sampled_slots } else: constraints = {} - - matching_items = item_collection.get_items_by_properties( - [ - SlotValueAnnotation(slot, value) - for slot, value in constraints.items() - ] - ) - - if matching_items: - target_item = max( - matching_items, - key=lambda item: preference_model.get_item_preference(item.id), + informable_slots = set(domain.get_informable_slots()).intersection( + target_item.properties.keys() ) - else: - target_item = item_collection.get_random_item() - if not constraints: - informable_slots = set(domain.get_informable_slots()).intersection( - target_item.properties.keys() - ) - if informable_slots: - num_constraints = random.randint(1, len(informable_slots)) - for slot in random.sample( - list(informable_slots), num_constraints - ): - constraints[slot] = target_item.get_property(slot) + if informable_slots: + num_constraints = random.randint(1, len(informable_slots)) + for slot in random.sample(list(informable_slots), num_constraints): + constraints[slot] = target_item.get_property(slot) requestable_slots = [ slot @@ -136,9 +139,9 @@ def generate_preference_grounded_information_need( class InformationNeed: - INCOMPLETE = "incomplete" - ATTEMPTED = "attempted" - COMPLETE = "complete" + SLOT_STATE_INCOMPLETE = "incomplete" + SLOT_STATE_ATTEMPTED = "attempted" + SLOT_STATE_COMPLETE = "complete" def __init__( self, @@ -164,12 +167,14 @@ def __init__( None, {slot: None for slot in requests} ) self.constraint_states = { - slot: (constraint_states or {}).get(slot, self.INCOMPLETE) + slot: (constraint_states or {}).get( + slot, self.SLOT_STATE_INCOMPLETE + ) for slot in constraints } self.request_states = { - slot: (request_states or {}).get(slot, self.INCOMPLETE) + slot: (request_states or {}).get(slot, self.SLOT_STATE_INCOMPLETE) for slot in requests } @@ -189,7 +194,7 @@ def get_requestable_slots(self) -> List[str]: return [ slot for slot in self.requested_slots - if self.request_states.get(slot) != self.COMPLETE + if self.request_states.get(slot) != self.SLOT_STATE_COMPLETE ] def _mark_state( @@ -201,46 +206,69 @@ def _mark_state( states: Mapping from slots to their current states. slot: Slot whose state should be updated. state: New state to assign to the slot. - - Returns: - None. """ if slot in states: states[slot] = state def mark_constraint_attempted(self, slot: str) -> None: - self._mark_state(self.constraint_states, slot, self.ATTEMPTED) + self._mark_state( + self.constraint_states, slot, self.SLOT_STATE_ATTEMPTED + ) def mark_constraint_complete(self, slot: str) -> None: - self._mark_state(self.constraint_states, slot, self.COMPLETE) + self._mark_state(self.constraint_states, slot, self.SLOT_STATE_COMPLETE) def mark_request_attempted(self, slot: str) -> None: - self._mark_state(self.request_states, slot, self.ATTEMPTED) + self._mark_state(self.request_states, slot, self.SLOT_STATE_ATTEMPTED) def mark_request_complete(self, slot: str, value: Any = None) -> None: if slot in self.request_states: self.requested_slots[slot] = value - self.request_states[slot] = self.COMPLETE + self.request_states[slot] = self.SLOT_STATE_COMPLETE - def get_goal_progress(self) -> Dict[str, int]: - states = [ + def _get_slot_progress(self, states: Dict[str, str]) -> Dict[str, int]: + """Returns status counts for a single slot category.""" + values = list(states.values()) + return { + self.SLOT_STATE_INCOMPLETE: values.count( + self.SLOT_STATE_INCOMPLETE + ), + self.SLOT_STATE_ATTEMPTED: values.count(self.SLOT_STATE_ATTEMPTED), + self.SLOT_STATE_COMPLETE: values.count(self.SLOT_STATE_COMPLETE), + "total": len(values), + } + + def get_information_need_progress(self) -> Dict[str, Dict[str, int]]: + """Returns progress counters for constraints and requests.""" + constraint_progress = self._get_slot_progress(self.constraint_states) + request_progress = self._get_slot_progress(self.request_states) + total_states = [ *self.constraint_states.values(), *self.request_states.values(), ] - return { - self.INCOMPLETE: states.count(self.INCOMPLETE), - self.ATTEMPTED: states.count(self.ATTEMPTED), - self.COMPLETE: states.count(self.COMPLETE), - "total": len(states), + "constraints": constraint_progress, + "requests": request_progress, + "overall": { + self.SLOT_STATE_INCOMPLETE: total_states.count( + self.SLOT_STATE_INCOMPLETE + ), + self.SLOT_STATE_ATTEMPTED: total_states.count( + self.SLOT_STATE_ATTEMPTED + ), + self.SLOT_STATE_COMPLETE: total_states.count( + self.SLOT_STATE_COMPLETE + ), + "total": len(total_states), + }, } - def get_goal_completion_ratio(self) -> float: - """Returns the fraction of completed goal components.""" - progress = self.get_goal_progress() + def get_information_need_completion_ratio(self) -> float: + """Returns the fraction of completed information-need components.""" + progress = self.get_information_need_progress()["overall"] if progress["total"] == 0: return 1.0 - return progress[self.COMPLETE] / progress["total"] + return progress[self.SLOT_STATE_COMPLETE] / progress["total"] @classmethod def from_dict(cls, data: Dict[str, Any]) -> InformationNeed: diff --git a/usersimcrs/simulator/agenda_based/interaction_model.py b/usersimcrs/simulator/agenda_based/interaction_model.py index 800d011f..71e8ec5e 100644 --- a/usersimcrs/simulator/agenda_based/interaction_model.py +++ b/usersimcrs/simulator/agenda_based/interaction_model.py @@ -315,7 +315,7 @@ def update_agenda( """ current_state = self.dialogue_state_tracker.get_current_state() agent_dialogue_acts = current_state.agent_dialogue_acts[-1] - self._update_goal_state_from_agent_dialogue_acts( + self._update_information_need_state_from_agent_dialogue_acts( information_need, agent_dialogue_acts ) user_dialogue_acts = [] @@ -358,12 +358,13 @@ def update_agenda( self.agenda.push_dialogue_acts(user_dialogue_acts) self.agenda.clean_agenda(information_need) - def _update_goal_state_from_agent_dialogue_acts( + def _update_information_need_state_from_agent_dialogue_acts( self, information_need: InformationNeed, agent_dialogue_acts: List[DialogueAct], ) -> None: - """Updates goal progress based on the latest agent dialogue acts. + """Updates information-need progress from the latest agent dialogue + acts. Marks requested slots as attempted or complete depending on whether the agent provided a value, and marks mentioned constraint slots as @@ -372,9 +373,6 @@ def _update_goal_state_from_agent_dialogue_acts( Args: information_need: Information need to update. agent_dialogue_acts: Latest dialogue acts produced by the agent. - - Returns: - None. """ for dialogue_act in agent_dialogue_acts: for annotation in dialogue_act.annotations: @@ -387,7 +385,13 @@ def _update_goal_state_from_agent_dialogue_acts( information_need.mark_request_attempted(slot) if slot in information_need.constraint_states: - information_need.mark_constraint_attempted(slot) + constraint_value = information_need.get_constraint_value( + slot + ) + if value == constraint_value: + information_need.mark_constraint_complete(slot) + else: + information_need.mark_constraint_attempted(slot) def _get_preference_intent( self, preference: float, preference_model: PreferenceModel @@ -459,7 +463,15 @@ def _generate_elicit_response_dialogue_acts( ) ) if elicited_slot in information_need.constraint_states: - information_need.mark_constraint_attempted(elicited_slot) + constraint_value = information_need.get_constraint_value( + elicited_slot + ) + if elicited_value == constraint_value: + information_need.mark_constraint_complete(elicited_slot) + else: + information_need.mark_constraint_attempted( + elicited_slot + ) else: # Agent is asking about value preferences on a given slot, e.g., # "What movie genre would you prefer?" The value is taken either diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index 7601141f..8dcb1099 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -64,7 +64,7 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._update_goal_state_from_agent_text(agent_utterance) + self._update_information_need_state_from_agent_text(agent_utterance) self.generation_prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index cf9f3e1b..e0d3403b 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -53,7 +53,7 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._update_goal_state_from_agent_text(agent_utterance) + self._update_information_need_state_from_agent_text(agent_utterance) self.prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index 05b6f501..dcdd7596 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -44,20 +44,81 @@ def _normalize_text(self, text: str) -> str: """ return " ".join(re.sub(r"[_-]", " ", text.lower()).split()) - def _update_goal_state_from_agent_text(self, utterance: Utterance) -> None: - """Updates goal state from a raw agent utterance.""" + def _looks_like_question(self, text: str) -> bool: + """Heuristically detects whether an utterance is a question. + + Args: + text: Input text. + + Returns: + True if the text appears to be a question, else False. + """ + stripped = text.strip() + if not stripped: + return False + + if stripped.endswith("?"): + return True + + if stripped.endswith((".", "!")): + return False + + normalized = self._normalize_text(stripped) + return bool( + re.match( + ( + r"^(who|what|when|where|why|how|which|" + r"is|are|was|were|do|does|did|can|could|" + r"would|will|should|have|has|had)\b" + ), + normalized, + ) + ) + + def _get_target_slot_values(self, slot: str) -> list[str]: + """Returns normalized values for a slot across target items. + + Args: + slot: Slot name. + + Returns: + List of normalized slot values. + """ + normalized_values: list[str] = [] + for item in self.information_need.target_items: + value = item.get_property(slot) + if value is None: + continue + values = value if isinstance(value, list) else [value] + normalized_values.extend( + self._normalize_text(str(entry)) for entry in values + ) + return normalized_values + + def _update_information_need_state_from_agent_text( + self, utterance: Utterance + ) -> None: + """Updates information-need state from a raw agent utterance. + + Args: + utterance: Agent utterance. + """ raw = getattr(utterance, "text", "") text = self._normalize_text(raw) if not text: return - is_question = "?" in raw + is_question = self._looks_like_question(raw) for slot in self.information_need.request_states: normalized_slot = self._normalize_text(slot) + target_values = self._get_target_slot_values(slot) + mentions_slot_or_value = normalized_slot in text or any( + value in text for value in target_values + ) - if normalized_slot not in text: + if not mentions_slot_or_value: continue if is_question: @@ -71,9 +132,12 @@ def _update_goal_state_from_agent_text(self, utterance: Utterance) -> None: normalized_values = [self._normalize_text(str(v)) for v in values] - if normalized_slot in text or any( - v in text for v in normalized_values - ): + if any(v in text for v in normalized_values): + if is_question: + self.information_need.mark_constraint_attempted(slot) + else: + self.information_need.mark_constraint_complete(slot) + elif normalized_slot in text: self.information_need.mark_constraint_attempted(slot) @abstractmethod From 4a05839824981657c6744bf75db14e59ffedc05b Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 30 Jun 2026 14:04:37 +0200 Subject: [PATCH 08/22] refactor names --- usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py | 2 +- usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py | 4 ++-- usersimcrs/simulator/user_simulator.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index 8dcb1099..642b52b7 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -64,7 +64,7 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._update_information_need_state_from_agent_text(agent_utterance) + self._update_information_need_state_from_utterance(agent_utterance) self.generation_prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index e0d3403b..03f39e31 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -53,12 +53,12 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._update_information_need_state_from_agent_text(agent_utterance) + self._update_information_need_state_from_utterance(agent_utterance) self.prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) user_utterance = self.llm_interface.generate_utterance(self.prompt) - self._update_goal_state_from_user_text(user_utterance) + self._update_information_need_state_from_utterance(user_utterance) self.prompt.update_prompt_context( user_utterance, DialogueParticipant.USER ) diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index dcdd7596..3bcf8144 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -95,7 +95,7 @@ def _get_target_slot_values(self, slot: str) -> list[str]: ) return normalized_values - def _update_information_need_state_from_agent_text( + def _update_information_need_state_from_utterance( self, utterance: Utterance ) -> None: """Updates information-need state from a raw agent utterance. From 25a37f4447a2dc81052a26c716f32411a9b74b32 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 7 Jul 2026 18:27:21 +0200 Subject: [PATCH 09/22] simplify model --- usersimcrs/core/information_need.py | 98 +++++++++++++---------------- 1 file changed, 44 insertions(+), 54 deletions(-) diff --git a/usersimcrs/core/information_need.py b/usersimcrs/core/information_need.py index c168eaa3..725c1271 100644 --- a/usersimcrs/core/information_need.py +++ b/usersimcrs/core/information_need.py @@ -13,10 +13,10 @@ from collections import defaultdict from typing import Any, Dict, List, Optional +from dialoguekit.core.slot_value_annotation import SlotValueAnnotation from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item import Item from usersimcrs.items.item_collection import ItemCollection -from dialoguekit.core.slot_value_annotation import SlotValueAnnotation from usersimcrs.user_modeling.preference_model import PreferenceModel @@ -63,10 +63,9 @@ def generate_preference_grounded_information_need( ) -> InformationNeed: """Generates an information need aligned with a preference model. - The function first identifies a target item aligned with the preference - model, then samples matching properties from that item as constraints. - This keeps the constraints grounded in a real item while still biasing - the information need toward the user's preferences. + The function first samples preferred slot-value pairs from the preference + model, then tries to find an item matching them. If no matching item is + found, the information need is returned without target items. Args: domain: Domain knowledge. @@ -76,66 +75,57 @@ def generate_preference_grounded_information_need( Returns: Information need. """ - preferred_constraints: Dict[str, str] = {} - for slot in domain.get_informable_slots(): - value, score = preference_model.get_slot_preference(slot) - if value is None or score < preference_model.PREFERENCE_THRESHOLD: - continue - preferred_constraints[slot] = value - - target_item = item_collection.get_random_item() - if preferred_constraints: - anchor_slot = random.choice(list(preferred_constraints.keys())) - anchor_value = preferred_constraints[anchor_slot] - matching_items = item_collection.get_items_by_properties( - [SlotValueAnnotation(anchor_slot, anchor_value)] - ) - liked_items = [ - item - for item in matching_items - if preference_model.get_item_preference(item.id) - >= preference_model.PREFERENCE_THRESHOLD + preferred_constraints = { + slot: value + for slot in domain.get_informable_slots() + for value, _ in [preference_model.get_slot_preference(slot)] + if value is not None + } + + matching_items = item_collection.get_items_by_properties( + [ + SlotValueAnnotation(slot, value) + for slot, value in preferred_constraints.items() ] + ) + target_items = [random.choice(matching_items)] if matching_items else [] - if liked_items: - target_item = random.choice(liked_items) - elif matching_items: - target_item = random.choice(matching_items) - - preference_aligned_slots = [ - slot - for slot, value in preferred_constraints.items() - if target_item.get_property(slot) == value - ] + constraint_source = ( + { + slot: target_items[0].get_property(slot) + for slot in domain.get_informable_slots() + if slot in target_items[0].properties + } + if target_items + else preferred_constraints + ) - if preference_aligned_slots: - num_constraints = random.randint(1, len(preference_aligned_slots)) - sampled_slots = random.sample(preference_aligned_slots, num_constraints) - constraints = { - slot: target_item.get_property(slot) for slot in sampled_slots + constraints = ( + { + slot: constraint_source[slot] + for slot in random.sample( + list(constraint_source), + random.randint(1, len(constraint_source)), + ) } - else: - constraints = {} - informable_slots = set(domain.get_informable_slots()).intersection( - target_item.properties.keys() - ) - if informable_slots: - num_constraints = random.randint(1, len(informable_slots)) - for slot in random.sample(list(informable_slots), num_constraints): - constraints[slot] = target_item.get_property(slot) + if constraint_source + else {} + ) requestable_slots = [ slot for slot in domain.get_requestable_slots() if slot not in constraints ] - if requestable_slots: - num_requests = random.randint(1, len(requestable_slots)) - requests = random.sample(requestable_slots, num_requests) - else: - requests = [] + requests = ( + random.sample( + requestable_slots, random.randint(1, len(requestable_slots)) + ) + if requestable_slots + else [] + ) - return InformationNeed([target_item], constraints, requests) + return InformationNeed(target_items, constraints, requests) class InformationNeed: From 3fba649c9857c96864bebebe4c19355484f580da Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 14 Jul 2026 11:58:23 +0200 Subject: [PATCH 10/22] add information need interface, simplify heuristic logic --- .../agenda_based/interaction_model.py | 35 ++--- .../simulator/information_need/__init__.py | 21 +++ .../information_need/heuristic_interface.py | 120 ++++++++++++++++++ .../simulator/information_need/interface.py | 56 ++++++++ .../information_need/llm_interface.py | 26 ++++ .../llm/llm_single_prompt_user_simulator.py | 1 - usersimcrs/simulator/user_simulator.py | 117 ++--------------- 7 files changed, 249 insertions(+), 127 deletions(-) create mode 100644 usersimcrs/simulator/information_need/__init__.py create mode 100644 usersimcrs/simulator/information_need/heuristic_interface.py create mode 100644 usersimcrs/simulator/information_need/interface.py create mode 100644 usersimcrs/simulator/information_need/llm_interface.py diff --git a/usersimcrs/simulator/agenda_based/interaction_model.py b/usersimcrs/simulator/agenda_based/interaction_model.py index 71e8ec5e..1df31658 100644 --- a/usersimcrs/simulator/agenda_based/interaction_model.py +++ b/usersimcrs/simulator/agenda_based/interaction_model.py @@ -27,6 +27,12 @@ ) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.simulator.agenda_based.agenda import Agenda +from usersimcrs.simulator.information_need.heuristic_interface import ( + HeuristicInformationNeedInterface, +) +from usersimcrs.simulator.information_need.interface import ( + apply_information_need_update, +) from usersimcrs.user_modeling.preference_model import PreferenceModel _LEMMATIZER = WordNetLemmatizer() @@ -75,6 +81,7 @@ def __init__( self._initialize_required_intents() self._domain = domain + self._information_need_interface = HeuristicInformationNeedInterface() ( self.transition_matrix_single, self.transition_matrix_compound, @@ -366,32 +373,16 @@ def _update_information_need_state_from_agent_dialogue_acts( """Updates information-need progress from the latest agent dialogue acts. - Marks requested slots as attempted or complete depending on whether - the agent provided a value, and marks mentioned constraint slots as - attempted. - Args: information_need: Information need to update. agent_dialogue_acts: Latest dialogue acts produced by the agent. """ - for dialogue_act in agent_dialogue_acts: - for annotation in dialogue_act.annotations: - slot = annotation.slot - value = annotation.value - if slot in information_need.request_states: - if value is not None: - information_need.mark_request_complete(slot, value) - else: - information_need.mark_request_attempted(slot) - - if slot in information_need.constraint_states: - constraint_value = information_need.get_constraint_value( - slot - ) - if value == constraint_value: - information_need.mark_constraint_complete(slot) - else: - information_need.mark_constraint_attempted(slot) + update = ( + self._information_need_interface.update_from_agent_dialogue_acts( + information_need, agent_dialogue_acts + ) + ) + apply_information_need_update(information_need, update) def _get_preference_intent( self, preference: float, preference_model: PreferenceModel diff --git a/usersimcrs/simulator/information_need/__init__.py b/usersimcrs/simulator/information_need/__init__.py new file mode 100644 index 00000000..c449c55b --- /dev/null +++ b/usersimcrs/simulator/information_need/__init__.py @@ -0,0 +1,21 @@ +"""Information-need update interfaces for simulators.""" + +from usersimcrs.simulator.information_need.heuristic_interface import ( + HeuristicInformationNeedInterface, +) +from usersimcrs.simulator.information_need.interface import ( + InformationNeedInterface, + SlotUpdate, + apply_information_need_update, +) +from usersimcrs.simulator.information_need.llm_interface import ( + LLMInformationNeedInterface, +) + +__all__ = [ + "InformationNeedInterface", + "SlotUpdate", + "LLMInformationNeedInterface", + "HeuristicInformationNeedInterface", + "apply_information_need_update", +] diff --git a/usersimcrs/simulator/information_need/heuristic_interface.py b/usersimcrs/simulator/information_need/heuristic_interface.py new file mode 100644 index 00000000..d858378c --- /dev/null +++ b/usersimcrs/simulator/information_need/heuristic_interface.py @@ -0,0 +1,120 @@ +"""Heuristic information-need update interface.""" + +from __future__ import annotations + +import re +from typing import List + +from dialoguekit.core.dialogue_act import DialogueAct +from dialoguekit.core.utterance import Utterance + +from usersimcrs.core.information_need import InformationNeed +from usersimcrs.simulator.information_need.interface import ( + InformationNeedInterface, + SlotUpdate, +) + + +class HeuristicInformationNeedInterface(InformationNeedInterface): + def _normalize_text(self, text: str) -> str: + return " ".join(re.sub(r"[_-]", " ", text.lower()).split()) + + def _get_target_slot_values( + self, information_need: InformationNeed, slot: str + ) -> List[str]: + normalized_values: List[str] = [] + for item in information_need.target_items: + value = item.get_property(slot) + if value is None: + continue + values = value if isinstance(value, list) else [value] + normalized_values.extend( + self._normalize_text(str(entry)) for entry in values + ) + return normalized_values + + def _slot_update( + self, kind: str, slot: str, is_complete: bool, value=None + ) -> SlotUpdate: + return SlotUpdate( + kind, + slot, + ( + InformationNeed.SLOT_STATE_COMPLETE + if is_complete + else InformationNeed.SLOT_STATE_ATTEMPTED + ), + value, + ) + + def update_from_utterance( + self, information_need: InformationNeed, utterance: Utterance + ) -> List[SlotUpdate]: + raw = getattr(utterance, "text", "") + text = self._normalize_text(raw) + + if not text: + return [] + + updates: List[SlotUpdate] = [] + + for slot in information_need.request_states: + normalized_slot = self._normalize_text(slot) + target_values = self._get_target_slot_values(information_need, slot) + value_mentioned = any(value in text for value in target_values) + slot_mentioned = normalized_slot in text + if slot_mentioned or value_mentioned: + updates.append( + self._slot_update( + "request", + slot, + value_mentioned, + raw.strip() if value_mentioned else None, + ) + ) + + for slot, value in information_need.constraints.items(): + normalized_slot = self._normalize_text(slot) + values = value if isinstance(value, list) else [value] + normalized_values = [self._normalize_text(str(v)) for v in values] + slot_mentioned = normalized_slot in text + value_mentioned = any(v in text for v in normalized_values) + + if slot_mentioned or value_mentioned: + updates.append( + self._slot_update( + "constraint", + slot, + value_mentioned, + ) + ) + + return updates + + def update_from_agent_dialogue_acts( + self, + information_need: InformationNeed, + agent_dialogue_acts: List[DialogueAct], + ) -> List[SlotUpdate]: + updates: List[SlotUpdate] = [] + for dialogue_act in agent_dialogue_acts: + for annotation in dialogue_act.annotations: + slot = annotation.slot + value = annotation.value + if slot in information_need.request_states: + updates.append( + self._slot_update( + "request", slot, value is not None, value + ) + ) + + if slot in information_need.constraint_states: + updates.append( + self._slot_update( + "constraint", + slot, + value + == information_need.get_constraint_value(slot), + ), + ) + return updates diff --git a/usersimcrs/simulator/information_need/interface.py b/usersimcrs/simulator/information_need/interface.py new file mode 100644 index 00000000..6164312d --- /dev/null +++ b/usersimcrs/simulator/information_need/interface.py @@ -0,0 +1,56 @@ +"""Information-need update interfaces.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, List + +from dialoguekit.core.dialogue_act import DialogueAct +from dialoguekit.core.utterance import Utterance + +from usersimcrs.core.information_need import InformationNeed + + +@dataclass +class SlotUpdate: + kind: str + slot: str + status: str + value: Any = None + + +class InformationNeedInterface(ABC): + @abstractmethod + def update_from_utterance( + self, information_need: InformationNeed, utterance: Utterance + ) -> List[SlotUpdate]: + """Builds information-need updates from an utterance.""" + raise NotImplementedError + + @abstractmethod + def update_from_agent_dialogue_acts( + self, + information_need: InformationNeed, + agent_dialogue_acts: List[DialogueAct], + ) -> List[SlotUpdate]: + """Builds information-need updates from agent dialogue acts.""" + raise NotImplementedError + + +def apply_information_need_update( + information_need: InformationNeed, updates: List[SlotUpdate] +) -> None: + """Applies structured updates to an information need.""" + for update in updates: + if update.kind == "request": + if update.status == InformationNeed.SLOT_STATE_COMPLETE: + information_need.mark_request_complete( + update.slot, update.value + ) + elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: + information_need.mark_request_attempted(update.slot) + elif update.status == InformationNeed.SLOT_STATE_COMPLETE: + information_need.mark_constraint_complete(update.slot) + elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: + information_need.mark_constraint_attempted(update.slot) diff --git a/usersimcrs/simulator/information_need/llm_interface.py b/usersimcrs/simulator/information_need/llm_interface.py new file mode 100644 index 00000000..334956dc --- /dev/null +++ b/usersimcrs/simulator/information_need/llm_interface.py @@ -0,0 +1,26 @@ +"""LLM-backed information-need update interface.""" + +from __future__ import annotations + +from dialoguekit.core.dialogue_act import DialogueAct +from dialoguekit.core.utterance import Utterance + +from usersimcrs.core.information_need import InformationNeed +from usersimcrs.simulator.information_need.interface import ( + InformationNeedInterface, + SlotUpdate, +) + + +class LLMInformationNeedInterface(InformationNeedInterface): + def update_from_utterance( + self, information_need: InformationNeed, utterance: Utterance + ) -> list[SlotUpdate]: + raise NotImplementedError + + def update_from_agent_dialogue_acts( + self, + information_need: InformationNeed, + agent_dialogue_acts: list[DialogueAct], + ) -> list[SlotUpdate]: + raise NotImplementedError diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index 03f39e31..a2dae7e7 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -58,7 +58,6 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: agent_utterance, DialogueParticipant.AGENT ) user_utterance = self.llm_interface.generate_utterance(self.prompt) - self._update_information_need_state_from_utterance(user_utterance) self.prompt.update_prompt_context( user_utterance, DialogueParticipant.USER ) diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index 3bcf8144..bc564738 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -1,7 +1,6 @@ """User simulator abstract class.""" from abc import ABC, abstractmethod -import re from dialoguekit.core.annotated_utterance import AnnotatedUtterance from dialoguekit.core.utterance import Utterance @@ -9,6 +8,12 @@ from usersimcrs.core.information_need import generate_random_information_need from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection +from usersimcrs.simulator.information_need.heuristic_interface import ( + HeuristicInformationNeedInterface, +) +from usersimcrs.simulator.information_need.interface import ( + apply_information_need_update, +) class UserSimulator(User, ABC): @@ -22,6 +27,7 @@ def __init__( super().__init__(id, UserType.SIMULATOR) self._domain = domain self._item_collection = item_collection + self._information_need_interface = HeuristicInformationNeedInterface() self.get_new_information_need() def get_new_information_need(self) -> None: @@ -30,115 +36,18 @@ def get_new_information_need(self) -> None: self._domain, self._item_collection ) - def _normalize_text(self, text: str) -> str: - """Normalizes text for simple slot and value matching. - - Lowercases the text, replaces underscores and hyphens with spaces, - and collapses repeated whitespace. - - Args: - text: Text to normalize. - - Returns: - Normalized text string. - """ - return " ".join(re.sub(r"[_-]", " ", text.lower()).split()) - - def _looks_like_question(self, text: str) -> bool: - """Heuristically detects whether an utterance is a question. - - Args: - text: Input text. - - Returns: - True if the text appears to be a question, else False. - """ - stripped = text.strip() - if not stripped: - return False - - if stripped.endswith("?"): - return True - - if stripped.endswith((".", "!")): - return False - - normalized = self._normalize_text(stripped) - return bool( - re.match( - ( - r"^(who|what|when|where|why|how|which|" - r"is|are|was|were|do|does|did|can|could|" - r"would|will|should|have|has|had)\b" - ), - normalized, - ) - ) - - def _get_target_slot_values(self, slot: str) -> list[str]: - """Returns normalized values for a slot across target items. - - Args: - slot: Slot name. - - Returns: - List of normalized slot values. - """ - normalized_values: list[str] = [] - for item in self.information_need.target_items: - value = item.get_property(slot) - if value is None: - continue - values = value if isinstance(value, list) else [value] - normalized_values.extend( - self._normalize_text(str(entry)) for entry in values - ) - return normalized_values - def _update_information_need_state_from_utterance( self, utterance: Utterance ) -> None: - """Updates information-need state from a raw agent utterance. + """Updates information-need state from an utterance. Args: - utterance: Agent utterance. + utterance: Utterance. """ - raw = getattr(utterance, "text", "") - text = self._normalize_text(raw) - - if not text: - return - - is_question = self._looks_like_question(raw) - - for slot in self.information_need.request_states: - normalized_slot = self._normalize_text(slot) - target_values = self._get_target_slot_values(slot) - mentions_slot_or_value = normalized_slot in text or any( - value in text for value in target_values - ) - - if not mentions_slot_or_value: - continue - - if is_question: - self.information_need.mark_request_attempted(slot) - else: - self.information_need.mark_request_complete(slot, raw.strip()) - - for slot, value in self.information_need.constraints.items(): - normalized_slot = self._normalize_text(slot) - values = value if isinstance(value, list) else [value] - - normalized_values = [self._normalize_text(str(v)) for v in values] - - if any(v in text for v in normalized_values): - if is_question: - self.information_need.mark_constraint_attempted(slot) - else: - self.information_need.mark_constraint_complete(slot) - elif normalized_slot in text: - self.information_need.mark_constraint_attempted(slot) + update = self._information_need_interface.update_from_utterance( + self.information_need, utterance + ) + apply_information_need_update(self.information_need, update) @abstractmethod def _generate_response(self, agent_utterance: Utterance) -> Utterance: From d2f3047fa5af978c1a48207c1ef6a804f86298ba Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 14 Jul 2026 12:25:30 +0200 Subject: [PATCH 11/22] remove redundunt functions --- .../agenda_based/interaction_model.py | 26 ++++--------------- .../simulator/information_need/__init__.py | 21 --------------- .../llm/llm_dual_prompt_user_simulator.py | 10 ++++++- .../llm/llm_single_prompt_user_simulator.py | 10 ++++++- usersimcrs/simulator/user_simulator.py | 16 ------------ 5 files changed, 23 insertions(+), 60 deletions(-) delete mode 100644 usersimcrs/simulator/information_need/__init__.py diff --git a/usersimcrs/simulator/agenda_based/interaction_model.py b/usersimcrs/simulator/agenda_based/interaction_model.py index 1df31658..ed1eec01 100644 --- a/usersimcrs/simulator/agenda_based/interaction_model.py +++ b/usersimcrs/simulator/agenda_based/interaction_model.py @@ -322,8 +322,11 @@ def update_agenda( """ current_state = self.dialogue_state_tracker.get_current_state() agent_dialogue_acts = current_state.agent_dialogue_acts[-1] - self._update_information_need_state_from_agent_dialogue_acts( - information_need, agent_dialogue_acts + apply_information_need_update( + information_need, + self._information_need_interface.update_from_agent_dialogue_acts( + information_need, agent_dialogue_acts + ), ) user_dialogue_acts = [] for dialogue_act in agent_dialogue_acts: @@ -365,25 +368,6 @@ def update_agenda( self.agenda.push_dialogue_acts(user_dialogue_acts) self.agenda.clean_agenda(information_need) - def _update_information_need_state_from_agent_dialogue_acts( - self, - information_need: InformationNeed, - agent_dialogue_acts: List[DialogueAct], - ) -> None: - """Updates information-need progress from the latest agent dialogue - acts. - - Args: - information_need: Information need to update. - agent_dialogue_acts: Latest dialogue acts produced by the agent. - """ - update = ( - self._information_need_interface.update_from_agent_dialogue_acts( - information_need, agent_dialogue_acts - ) - ) - apply_information_need_update(information_need, update) - def _get_preference_intent( self, preference: float, preference_model: PreferenceModel ) -> Intent: diff --git a/usersimcrs/simulator/information_need/__init__.py b/usersimcrs/simulator/information_need/__init__.py deleted file mode 100644 index c449c55b..00000000 --- a/usersimcrs/simulator/information_need/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Information-need update interfaces for simulators.""" - -from usersimcrs.simulator.information_need.heuristic_interface import ( - HeuristicInformationNeedInterface, -) -from usersimcrs.simulator.information_need.interface import ( - InformationNeedInterface, - SlotUpdate, - apply_information_need_update, -) -from usersimcrs.simulator.information_need.llm_interface import ( - LLMInformationNeedInterface, -) - -__all__ = [ - "InformationNeedInterface", - "SlotUpdate", - "LLMInformationNeedInterface", - "HeuristicInformationNeedInterface", - "apply_information_need_update", -] diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index 642b52b7..fba19aed 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -10,6 +10,9 @@ from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface +from usersimcrs.simulator.information_need.interface import ( + apply_information_need_update, +) from usersimcrs.simulator.llm.prompt.stop_prompt import ( DEFAULT_STOP_DEFINITION, StopPrompt, @@ -64,7 +67,12 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._update_information_need_state_from_utterance(agent_utterance) + apply_information_need_update( + self.information_need, + self._information_need_interface.update_from_utterance( + self.information_need, agent_utterance + ), + ) self.generation_prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index a2dae7e7..e41e4854 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -9,6 +9,9 @@ from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface +from usersimcrs.simulator.information_need.interface import ( + apply_information_need_update, +) from usersimcrs.simulator.llm.prompt.utterance_generation_prompt import ( DEFAULT_TASK_DEFINITION, UtteranceGenerationPrompt, @@ -53,7 +56,12 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._update_information_need_state_from_utterance(agent_utterance) + apply_information_need_update( + self.information_need, + self._information_need_interface.update_from_utterance( + self.information_need, agent_utterance + ), + ) self.prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index bc564738..b95c9bb9 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -11,9 +11,6 @@ from usersimcrs.simulator.information_need.heuristic_interface import ( HeuristicInformationNeedInterface, ) -from usersimcrs.simulator.information_need.interface import ( - apply_information_need_update, -) class UserSimulator(User, ABC): @@ -36,19 +33,6 @@ def get_new_information_need(self) -> None: self._domain, self._item_collection ) - def _update_information_need_state_from_utterance( - self, utterance: Utterance - ) -> None: - """Updates information-need state from an utterance. - - Args: - utterance: Utterance. - """ - update = self._information_need_interface.update_from_utterance( - self.information_need, utterance - ) - apply_information_need_update(self.information_need, update) - @abstractmethod def _generate_response(self, agent_utterance: Utterance) -> Utterance: """Generates response to the agent utterance. From 159ffac5026898cd0822fd909414fad8fe0ab590 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 14 Jul 2026 13:31:45 +0200 Subject: [PATCH 12/22] add comments --- .../information_need/heuristic_interface.py | 20 ++++++++++++++ .../simulator/information_need/interface.py | 27 ++++++++++++++++--- .../llm/llm_dual_prompt_user_simulator.py | 4 +++ .../llm/llm_single_prompt_user_simulator.py | 4 +++ usersimcrs/simulator/user_simulator.py | 4 --- 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/usersimcrs/simulator/information_need/heuristic_interface.py b/usersimcrs/simulator/information_need/heuristic_interface.py index d858378c..2555338e 100644 --- a/usersimcrs/simulator/information_need/heuristic_interface.py +++ b/usersimcrs/simulator/information_need/heuristic_interface.py @@ -22,6 +22,15 @@ def _normalize_text(self, text: str) -> str: def _get_target_slot_values( self, information_need: InformationNeed, slot: str ) -> List[str]: + """Collects normalized target values for a slot. + + Args: + information_need: Information need with target items. + slot: Slot name. + + Return: + Normalized target values. + """ normalized_values: List[str] = [] for item in information_need.target_items: value = item.get_property(slot) @@ -36,6 +45,17 @@ def _get_target_slot_values( def _slot_update( self, kind: str, slot: str, is_complete: bool, value=None ) -> SlotUpdate: + """Builds a slot update with complete or attempted status. + + Args: + kind: Slot kind, either request or constraint. + slot: Slot name. + is_complete: Whether the slot is complete. + value: Slot value for request completion. + + Return: + Slot update. + """ return SlotUpdate( kind, slot, diff --git a/usersimcrs/simulator/information_need/interface.py b/usersimcrs/simulator/information_need/interface.py index 6164312d..c246c524 100644 --- a/usersimcrs/simulator/information_need/interface.py +++ b/usersimcrs/simulator/information_need/interface.py @@ -25,7 +25,15 @@ class InformationNeedInterface(ABC): def update_from_utterance( self, information_need: InformationNeed, utterance: Utterance ) -> List[SlotUpdate]: - """Builds information-need updates from an utterance.""" + """Builds information-need updates from an utterance. + + Args: + information_need: Information need to update. + utterance: Utterance to inspect. + + Return: + List of slot updates. + """ raise NotImplementedError @abstractmethod @@ -34,14 +42,27 @@ def update_from_agent_dialogue_acts( information_need: InformationNeed, agent_dialogue_acts: List[DialogueAct], ) -> List[SlotUpdate]: - """Builds information-need updates from agent dialogue acts.""" + """Builds information-need updates from agent dialogue acts. + + Args: + information_need: Information need to update. + agent_dialogue_acts: Agent dialogue acts to inspect. + + Return: + List of slot updates. + """ raise NotImplementedError def apply_information_need_update( information_need: InformationNeed, updates: List[SlotUpdate] ) -> None: - """Applies structured updates to an information need.""" + """Applies slot updates to an information need. + + Args: + information_need: Information need to update. + updates: Slot updates to apply. + """ for update in updates: if update.kind == "request": if update.status == InformationNeed.SLOT_STATE_COMPLETE: diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index fba19aed..aa67301a 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -10,6 +10,9 @@ from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface +from usersimcrs.simulator.information_need.heuristic_interface import ( + HeuristicInformationNeedInterface, +) from usersimcrs.simulator.information_need.interface import ( apply_information_need_update, ) @@ -50,6 +53,7 @@ def __init__( persona: Persona of the user. Defaults to None. """ super().__init__(id, domain, item_collection) + self._information_need_interface = HeuristicInformationNeedInterface() self.llm_interface = llm_interface self.generation_prompt = UtteranceGenerationPrompt( self.information_need, item_type, task_definition, persona diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index e41e4854..9c0987a5 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -9,6 +9,9 @@ from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface +from usersimcrs.simulator.information_need.heuristic_interface import ( + HeuristicInformationNeedInterface, +) from usersimcrs.simulator.information_need.interface import ( apply_information_need_update, ) @@ -42,6 +45,7 @@ def __init__( persona: Persona of the user. Defaults to None. """ super().__init__(id, domain, item_collection) + self._information_need_interface = HeuristicInformationNeedInterface() self.llm_interface = llm_interface self.prompt = UtteranceGenerationPrompt( self.information_need, item_type, task_definition, persona diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index b95c9bb9..af238c4e 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -8,9 +8,6 @@ from usersimcrs.core.information_need import generate_random_information_need from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection -from usersimcrs.simulator.information_need.heuristic_interface import ( - HeuristicInformationNeedInterface, -) class UserSimulator(User, ABC): @@ -24,7 +21,6 @@ def __init__( super().__init__(id, UserType.SIMULATOR) self._domain = domain self._item_collection = item_collection - self._information_need_interface = HeuristicInformationNeedInterface() self.get_new_information_need() def get_new_information_need(self) -> None: From a93b00cb2c2e1dd3d79547e20466f3cd3d2c52b1 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 14 Jul 2026 13:36:15 +0200 Subject: [PATCH 13/22] remove info functions --- usersimcrs/core/information_need.py | 44 ----------------------------- 1 file changed, 44 deletions(-) diff --git a/usersimcrs/core/information_need.py b/usersimcrs/core/information_need.py index 725c1271..44f34c2c 100644 --- a/usersimcrs/core/information_need.py +++ b/usersimcrs/core/information_need.py @@ -216,50 +216,6 @@ def mark_request_complete(self, slot: str, value: Any = None) -> None: self.requested_slots[slot] = value self.request_states[slot] = self.SLOT_STATE_COMPLETE - def _get_slot_progress(self, states: Dict[str, str]) -> Dict[str, int]: - """Returns status counts for a single slot category.""" - values = list(states.values()) - return { - self.SLOT_STATE_INCOMPLETE: values.count( - self.SLOT_STATE_INCOMPLETE - ), - self.SLOT_STATE_ATTEMPTED: values.count(self.SLOT_STATE_ATTEMPTED), - self.SLOT_STATE_COMPLETE: values.count(self.SLOT_STATE_COMPLETE), - "total": len(values), - } - - def get_information_need_progress(self) -> Dict[str, Dict[str, int]]: - """Returns progress counters for constraints and requests.""" - constraint_progress = self._get_slot_progress(self.constraint_states) - request_progress = self._get_slot_progress(self.request_states) - total_states = [ - *self.constraint_states.values(), - *self.request_states.values(), - ] - return { - "constraints": constraint_progress, - "requests": request_progress, - "overall": { - self.SLOT_STATE_INCOMPLETE: total_states.count( - self.SLOT_STATE_INCOMPLETE - ), - self.SLOT_STATE_ATTEMPTED: total_states.count( - self.SLOT_STATE_ATTEMPTED - ), - self.SLOT_STATE_COMPLETE: total_states.count( - self.SLOT_STATE_COMPLETE - ), - "total": len(total_states), - }, - } - - def get_information_need_completion_ratio(self) -> float: - """Returns the fraction of completed information-need components.""" - progress = self.get_information_need_progress()["overall"] - if progress["total"] == 0: - return 1.0 - return progress[self.SLOT_STATE_COMPLETE] / progress["total"] - @classmethod def from_dict(cls, data: Dict[str, Any]) -> InformationNeed: """Creates information need from a dictionary.""" From d5bffa38968f23540015a9211950435d07149eac Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 14 Jul 2026 16:48:49 +0200 Subject: [PATCH 14/22] add init file --- .../simulator/information_need/__init__.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 usersimcrs/simulator/information_need/__init__.py diff --git a/usersimcrs/simulator/information_need/__init__.py b/usersimcrs/simulator/information_need/__init__.py new file mode 100644 index 00000000..e52f0e2c --- /dev/null +++ b/usersimcrs/simulator/information_need/__init__.py @@ -0,0 +1,17 @@ +"""Information-need interfaces and helpers.""" + +from .heuristic_interface import HeuristicInformationNeedInterface +from .interface import ( + InformationNeedInterface, + SlotUpdate, + apply_information_need_update, +) +from .llm_interface import LLMInformationNeedInterface + +__all__ = [ + "HeuristicInformationNeedInterface", + "InformationNeedInterface", + "LLMInformationNeedInterface", + "SlotUpdate", + "apply_information_need_update", +] From 8acb1eb964f1adefabee13ab012777fc39cda9f0 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 21 Jul 2026 10:41:56 +0200 Subject: [PATCH 15/22] refactor information need --- usersimcrs/core/information_need.py | 44 ++++++---- .../agenda_based/interaction_model.py | 20 +++-- .../simulator/information_need/__init__.py | 18 ++--- .../information_need/heuristic_interface.py | 32 +++----- .../information_need_tracker.py | 81 +++++++++++++++++++ .../simulator/information_need/interface.py | 77 ------------------ .../information_need/llm_interface.py | 14 ++-- .../llm/llm_dual_prompt_user_simulator.py | 20 +++-- .../llm/llm_single_prompt_user_simulator.py | 20 +++-- 9 files changed, 162 insertions(+), 164 deletions(-) create mode 100644 usersimcrs/simulator/information_need/information_need_tracker.py delete mode 100644 usersimcrs/simulator/information_need/interface.py diff --git a/usersimcrs/core/information_need.py b/usersimcrs/core/information_need.py index 44f34c2c..919e6f96 100644 --- a/usersimcrs/core/information_need.py +++ b/usersimcrs/core/information_need.py @@ -11,7 +11,7 @@ import random from collections import defaultdict -from typing import Any, Dict, List, Optional +from typing import Any, DefaultDict, Dict, List, Optional from dialoguekit.core.slot_value_annotation import SlotValueAnnotation from usersimcrs.core.simulation_domain import SimulationDomain @@ -156,17 +156,25 @@ def __init__( self.requested_slots = defaultdict( None, {slot: None for slot in requests} ) - self.constraint_states = { - slot: (constraint_states or {}).get( - slot, self.SLOT_STATE_INCOMPLETE - ) - for slot in constraints - } + self.constraint_states: DefaultDict[str, str] = defaultdict( + lambda: self.SLOT_STATE_INCOMPLETE, + { + slot: (constraint_states or {}).get( + slot, self.SLOT_STATE_INCOMPLETE + ) + for slot in constraints + }, + ) - self.request_states = { - slot: (request_states or {}).get(slot, self.SLOT_STATE_INCOMPLETE) - for slot in requests - } + self.request_states: DefaultDict[str, str] = defaultdict( + lambda: self.SLOT_STATE_INCOMPLETE, + { + slot: (request_states or {}).get( + slot, self.SLOT_STATE_INCOMPLETE + ) + for slot in requests + }, + ) def get_constraint_value(self, slot: str) -> Any: """Returns the value of a constraint slot. @@ -201,17 +209,21 @@ def _mark_state( states[slot] = state def mark_constraint_attempted(self, slot: str) -> None: + """Marks a constraint slot as attempted.""" self._mark_state( self.constraint_states, slot, self.SLOT_STATE_ATTEMPTED ) def mark_constraint_complete(self, slot: str) -> None: + """Marks a constraint slot as complete.""" self._mark_state(self.constraint_states, slot, self.SLOT_STATE_COMPLETE) def mark_request_attempted(self, slot: str) -> None: + """Marks a request slot as attempted.""" self._mark_state(self.request_states, slot, self.SLOT_STATE_ATTEMPTED) - def mark_request_complete(self, slot: str, value: Any = None) -> None: + def mark_request_complete(self, slot: str, value: Any) -> None: + """Marks a request slot as complete with the observed value.""" if slot in self.request_states: self.requested_slots[slot] = value self.request_states[slot] = self.SLOT_STATE_COMPLETE @@ -224,8 +236,8 @@ def from_dict(cls, data: Dict[str, Any]) -> InformationNeed: target_items=target_items, constraints=data["constraints"], requests=data["requests"], - constraint_states=data.get("constraint_states"), - request_states=data.get("request_states"), + constraint_states=data.get("constraint_states", {}), + request_states=data.get("request_states", {}), ) def to_dict(self) -> Dict[str, Any]: @@ -237,6 +249,6 @@ def to_dict(self) -> Dict[str, Any]: ], "constraints": self.constraints, "requests": list(self.requested_slots.keys()), - "constraint_states": dict(self.constraint_states), - "request_states": dict(self.request_states), + "constraint_states": self.constraint_states, + "request_states": self.request_states, } diff --git a/usersimcrs/simulator/agenda_based/interaction_model.py b/usersimcrs/simulator/agenda_based/interaction_model.py index ed1eec01..3ef58a90 100644 --- a/usersimcrs/simulator/agenda_based/interaction_model.py +++ b/usersimcrs/simulator/agenda_based/interaction_model.py @@ -27,11 +27,8 @@ ) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.simulator.agenda_based.agenda import Agenda -from usersimcrs.simulator.information_need.heuristic_interface import ( - HeuristicInformationNeedInterface, -) -from usersimcrs.simulator.information_need.interface import ( - apply_information_need_update, +from usersimcrs.simulator.information_need import ( + HeuristicInformationNeedTracker, ) from usersimcrs.user_modeling.preference_model import PreferenceModel @@ -81,7 +78,6 @@ def __init__( self._initialize_required_intents() self._domain = domain - self._information_need_interface = HeuristicInformationNeedInterface() ( self.transition_matrix_single, self.transition_matrix_compound, @@ -322,11 +318,13 @@ def update_agenda( """ current_state = self.dialogue_state_tracker.get_current_state() agent_dialogue_acts = current_state.agent_dialogue_acts[-1] - apply_information_need_update( - information_need, - self._information_need_interface.update_from_agent_dialogue_acts( - information_need, agent_dialogue_acts - ), + information_need_tracker = HeuristicInformationNeedTracker( + information_need + ) + information_need_tracker.apply_updates( + information_need_tracker.update_from_agent_dialogue_acts( + agent_dialogue_acts + ) ) user_dialogue_acts = [] for dialogue_act in agent_dialogue_acts: diff --git a/usersimcrs/simulator/information_need/__init__.py b/usersimcrs/simulator/information_need/__init__.py index e52f0e2c..68fec1ab 100644 --- a/usersimcrs/simulator/information_need/__init__.py +++ b/usersimcrs/simulator/information_need/__init__.py @@ -1,17 +1,15 @@ -"""Information-need interfaces and helpers.""" +"""Information need trackers.""" -from .heuristic_interface import HeuristicInformationNeedInterface -from .interface import ( - InformationNeedInterface, +from .heuristic_interface import HeuristicInformationNeedTracker +from .information_need_tracker import ( + InformationNeedTracker, SlotUpdate, - apply_information_need_update, ) -from .llm_interface import LLMInformationNeedInterface +from .llm_interface import LLMInformationNeedTracker __all__ = [ - "HeuristicInformationNeedInterface", - "InformationNeedInterface", - "LLMInformationNeedInterface", + "HeuristicInformationNeedTracker", + "InformationNeedTracker", + "LLMInformationNeedTracker", "SlotUpdate", - "apply_information_need_update", ] diff --git a/usersimcrs/simulator/information_need/heuristic_interface.py b/usersimcrs/simulator/information_need/heuristic_interface.py index 2555338e..22e676eb 100644 --- a/usersimcrs/simulator/information_need/heuristic_interface.py +++ b/usersimcrs/simulator/information_need/heuristic_interface.py @@ -1,4 +1,4 @@ -"""Heuristic information-need update interface.""" +"""Heuristic information need tracker.""" from __future__ import annotations @@ -9,30 +9,27 @@ from dialoguekit.core.utterance import Utterance from usersimcrs.core.information_need import InformationNeed -from usersimcrs.simulator.information_need.interface import ( - InformationNeedInterface, +from usersimcrs.simulator.information_need.information_need_tracker import ( + InformationNeedTracker, SlotUpdate, ) -class HeuristicInformationNeedInterface(InformationNeedInterface): +class HeuristicInformationNeedTracker(InformationNeedTracker): def _normalize_text(self, text: str) -> str: return " ".join(re.sub(r"[_-]", " ", text.lower()).split()) - def _get_target_slot_values( - self, information_need: InformationNeed, slot: str - ) -> List[str]: + def _get_target_slot_values(self, slot: str) -> List[str]: """Collects normalized target values for a slot. Args: - information_need: Information need with target items. slot: Slot name. Return: Normalized target values. """ normalized_values: List[str] = [] - for item in information_need.target_items: + for item in self.information_need.target_items: value = item.get_property(slot) if value is None: continue @@ -67,9 +64,7 @@ def _slot_update( value, ) - def update_from_utterance( - self, information_need: InformationNeed, utterance: Utterance - ) -> List[SlotUpdate]: + def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: raw = getattr(utterance, "text", "") text = self._normalize_text(raw) @@ -78,9 +73,9 @@ def update_from_utterance( updates: List[SlotUpdate] = [] - for slot in information_need.request_states: + for slot in self.information_need.request_states: normalized_slot = self._normalize_text(slot) - target_values = self._get_target_slot_values(information_need, slot) + target_values = self._get_target_slot_values(slot) value_mentioned = any(value in text for value in target_values) slot_mentioned = normalized_slot in text if slot_mentioned or value_mentioned: @@ -93,7 +88,7 @@ def update_from_utterance( ) ) - for slot, value in information_need.constraints.items(): + for slot, value in self.information_need.constraints.items(): normalized_slot = self._normalize_text(slot) values = value if isinstance(value, list) else [value] normalized_values = [self._normalize_text(str(v)) for v in values] @@ -113,7 +108,6 @@ def update_from_utterance( def update_from_agent_dialogue_acts( self, - information_need: InformationNeed, agent_dialogue_acts: List[DialogueAct], ) -> List[SlotUpdate]: updates: List[SlotUpdate] = [] @@ -121,20 +115,20 @@ def update_from_agent_dialogue_acts( for annotation in dialogue_act.annotations: slot = annotation.slot value = annotation.value - if slot in information_need.request_states: + if slot in self.information_need.request_states: updates.append( self._slot_update( "request", slot, value is not None, value ) ) - if slot in information_need.constraint_states: + if slot in self.information_need.constraint_states: updates.append( self._slot_update( "constraint", slot, value - == information_need.get_constraint_value(slot), + == self.information_need.get_constraint_value(slot), ), ) return updates diff --git a/usersimcrs/simulator/information_need/information_need_tracker.py b/usersimcrs/simulator/information_need/information_need_tracker.py new file mode 100644 index 00000000..0745df80 --- /dev/null +++ b/usersimcrs/simulator/information_need/information_need_tracker.py @@ -0,0 +1,81 @@ +"""Information need tracker. + +It updates the state of the information need during the conversation. That is, +tracking the state (incomplete, attempted, complete) of the constraints and +requests. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, List + +from dialoguekit.core.dialogue_act import DialogueAct +from dialoguekit.core.utterance import Utterance + +from usersimcrs.core.information_need import InformationNeed + + +@dataclass +class SlotUpdate: + kind: str + slot: str + status: str + value: Any = None + + +class InformationNeedTracker(ABC): + """Tracks conversation progress against an information need.""" + + def __init__(self, information_need: InformationNeed) -> None: + """Initializes the tracker with the information need to update.""" + self.information_need = information_need + + @abstractmethod + def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: + """Builds information-need updates from an utterance. + + Args: + utterance: Utterance to inspect. + + Return: + List of slot updates. + """ + raise NotImplementedError + + @abstractmethod + def update_from_agent_dialogue_acts( + self, agent_dialogue_acts: List[DialogueAct] + ) -> List[SlotUpdate]: + """Builds information-need updates from agent dialogue acts. + + Args: + agent_dialogue_acts: Agent dialogue acts to inspect. + + Return: + List of slot updates. + """ + raise NotImplementedError + + def apply_updates(self, updates: List[SlotUpdate]) -> None: + """Applies slot updates to the tracked information need.""" + self.apply_updates_to_information_need(self.information_need, updates) + + @staticmethod + def apply_updates_to_information_need( + information_need: InformationNeed, updates: List[SlotUpdate] + ) -> None: + """Applies slot updates to an information need.""" + for update in updates: + if update.kind == "request": + if update.status == InformationNeed.SLOT_STATE_COMPLETE: + information_need.mark_request_complete( + update.slot, update.value + ) + elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: + information_need.mark_request_attempted(update.slot) + elif update.status == InformationNeed.SLOT_STATE_COMPLETE: + information_need.mark_constraint_complete(update.slot) + elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: + information_need.mark_constraint_attempted(update.slot) diff --git a/usersimcrs/simulator/information_need/interface.py b/usersimcrs/simulator/information_need/interface.py deleted file mode 100644 index c246c524..00000000 --- a/usersimcrs/simulator/information_need/interface.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Information-need update interfaces.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import Any, List - -from dialoguekit.core.dialogue_act import DialogueAct -from dialoguekit.core.utterance import Utterance - -from usersimcrs.core.information_need import InformationNeed - - -@dataclass -class SlotUpdate: - kind: str - slot: str - status: str - value: Any = None - - -class InformationNeedInterface(ABC): - @abstractmethod - def update_from_utterance( - self, information_need: InformationNeed, utterance: Utterance - ) -> List[SlotUpdate]: - """Builds information-need updates from an utterance. - - Args: - information_need: Information need to update. - utterance: Utterance to inspect. - - Return: - List of slot updates. - """ - raise NotImplementedError - - @abstractmethod - def update_from_agent_dialogue_acts( - self, - information_need: InformationNeed, - agent_dialogue_acts: List[DialogueAct], - ) -> List[SlotUpdate]: - """Builds information-need updates from agent dialogue acts. - - Args: - information_need: Information need to update. - agent_dialogue_acts: Agent dialogue acts to inspect. - - Return: - List of slot updates. - """ - raise NotImplementedError - - -def apply_information_need_update( - information_need: InformationNeed, updates: List[SlotUpdate] -) -> None: - """Applies slot updates to an information need. - - Args: - information_need: Information need to update. - updates: Slot updates to apply. - """ - for update in updates: - if update.kind == "request": - if update.status == InformationNeed.SLOT_STATE_COMPLETE: - information_need.mark_request_complete( - update.slot, update.value - ) - elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: - information_need.mark_request_attempted(update.slot) - elif update.status == InformationNeed.SLOT_STATE_COMPLETE: - information_need.mark_constraint_complete(update.slot) - elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: - information_need.mark_constraint_attempted(update.slot) diff --git a/usersimcrs/simulator/information_need/llm_interface.py b/usersimcrs/simulator/information_need/llm_interface.py index 334956dc..ded539a4 100644 --- a/usersimcrs/simulator/information_need/llm_interface.py +++ b/usersimcrs/simulator/information_need/llm_interface.py @@ -1,26 +1,22 @@ -"""LLM-backed information-need update interface.""" +"""LLM-backed information need tracker.""" from __future__ import annotations from dialoguekit.core.dialogue_act import DialogueAct from dialoguekit.core.utterance import Utterance -from usersimcrs.core.information_need import InformationNeed -from usersimcrs.simulator.information_need.interface import ( - InformationNeedInterface, +from usersimcrs.simulator.information_need.information_need_tracker import ( + InformationNeedTracker, SlotUpdate, ) -class LLMInformationNeedInterface(InformationNeedInterface): - def update_from_utterance( - self, information_need: InformationNeed, utterance: Utterance - ) -> list[SlotUpdate]: +class LLMInformationNeedTracker(InformationNeedTracker): + def update_from_utterance(self, utterance: Utterance) -> list[SlotUpdate]: raise NotImplementedError def update_from_agent_dialogue_acts( self, - information_need: InformationNeed, agent_dialogue_acts: list[DialogueAct], ) -> list[SlotUpdate]: raise NotImplementedError diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index aa67301a..abf7d9ec 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -10,11 +10,8 @@ from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface -from usersimcrs.simulator.information_need.heuristic_interface import ( - HeuristicInformationNeedInterface, -) -from usersimcrs.simulator.information_need.interface import ( - apply_information_need_update, +from usersimcrs.simulator.information_need import ( + HeuristicInformationNeedTracker, ) from usersimcrs.simulator.llm.prompt.stop_prompt import ( DEFAULT_STOP_DEFINITION, @@ -53,7 +50,9 @@ def __init__( persona: Persona of the user. Defaults to None. """ super().__init__(id, domain, item_collection) - self._information_need_interface = HeuristicInformationNeedInterface() + self._information_need_tracker = HeuristicInformationNeedTracker( + self.information_need + ) self.llm_interface = llm_interface self.generation_prompt = UtteranceGenerationPrompt( self.information_need, item_type, task_definition, persona @@ -71,11 +70,10 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - apply_information_need_update( - self.information_need, - self._information_need_interface.update_from_utterance( - self.information_need, agent_utterance - ), + self._information_need_tracker.apply_updates( + self._information_need_tracker.update_from_utterance( + agent_utterance + ) ) self.generation_prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index 9c0987a5..51529a6f 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -9,11 +9,8 @@ from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface -from usersimcrs.simulator.information_need.heuristic_interface import ( - HeuristicInformationNeedInterface, -) -from usersimcrs.simulator.information_need.interface import ( - apply_information_need_update, +from usersimcrs.simulator.information_need import ( + HeuristicInformationNeedTracker, ) from usersimcrs.simulator.llm.prompt.utterance_generation_prompt import ( DEFAULT_TASK_DEFINITION, @@ -45,7 +42,9 @@ def __init__( persona: Persona of the user. Defaults to None. """ super().__init__(id, domain, item_collection) - self._information_need_interface = HeuristicInformationNeedInterface() + self._information_need_tracker = HeuristicInformationNeedTracker( + self.information_need + ) self.llm_interface = llm_interface self.prompt = UtteranceGenerationPrompt( self.information_need, item_type, task_definition, persona @@ -60,11 +59,10 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - apply_information_need_update( - self.information_need, - self._information_need_interface.update_from_utterance( - self.information_need, agent_utterance - ), + self._information_need_tracker.apply_updates( + self._information_need_tracker.update_from_utterance( + agent_utterance + ) ) self.prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT From 635c8273922f04ace3a9a6dcde1520904c85c61b Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 21 Jul 2026 13:48:17 +0200 Subject: [PATCH 16/22] move imformation need and add tracker --- .../information_need_annotator.py | 4 ++- tests/conftest.py | 4 ++- tests/core/test_information_need.py | 2 +- .../agenda_based/test_interaction_model.py | 13 ++++++-- tests/simulator/llm/test_stop_prompt.py | 4 ++- .../llm/test_utterance_generation_prompt.py | 4 ++- usersimcrs/core/__init__.py | 2 -- .../information_need_management/__init__.py | 19 ++++++++++++ .../information_need.py | 0 .../information_need_tracker.py | 31 +++++++++++-------- .../information_need_trackers}/__init__.py | 2 +- .../heuristic_interface.py | 21 ++++++++----- .../llm_interface.py | 2 +- usersimcrs/simulator/agenda_based/agenda.py | 4 ++- .../agenda_based/agenda_based_simulator.py | 4 +-- .../agenda_based/interaction_model.py | 30 +++++++++--------- .../llm/llm_dual_prompt_user_simulator.py | 22 ++++++------- .../llm/llm_single_prompt_user_simulator.py | 17 +++++----- usersimcrs/simulator/llm/prompt/prompt.py | 4 ++- .../simulator/llm/prompt/stop_prompt.py | 4 ++- .../llm/prompt/utterance_generation_prompt.py | 4 ++- usersimcrs/simulator/user_simulator.py | 13 ++++++-- 22 files changed, 133 insertions(+), 77 deletions(-) create mode 100644 usersimcrs/information_need_management/__init__.py rename usersimcrs/{core => information_need_management}/information_need.py (100%) rename usersimcrs/{simulator/information_need => information_need_management}/information_need_tracker.py (75%) rename usersimcrs/{simulator/information_need => information_need_management/information_need_trackers}/__init__.py (81%) rename usersimcrs/{simulator/information_need => information_need_management/information_need_trackers}/heuristic_interface.py (84%) rename usersimcrs/{simulator/information_need => information_need_management/information_need_trackers}/llm_interface.py (87%) diff --git a/scripts/datasets/information_need_annotation/information_need_annotator.py b/scripts/datasets/information_need_annotation/information_need_annotator.py index c7076a61..ce5c0368 100644 --- a/scripts/datasets/information_need_annotation/information_need_annotator.py +++ b/scripts/datasets/information_need_annotation/information_need_annotator.py @@ -10,7 +10,9 @@ import os from dialoguekit.core.dialogue import Dialogue -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.llm_interfaces.llm_interface import LLMInterface DEFAULT_INITIAL_PROMPT_MOVIES_FILE = "scripts/datasets/information_need_annotation/information_need_prompt_movies_default.txt" # noqa: E501 diff --git a/tests/conftest.py b/tests/conftest.py index 81a025b9..0fd05314 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,9 @@ from dialoguekit.core.intent import Intent from dialoguekit.utils.dialogue_reader import json_to_dialogues -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item import Item from usersimcrs.items.item_collection import ItemCollection diff --git a/tests/core/test_information_need.py b/tests/core/test_information_need.py index 5ab7c62d..db4b5fbb 100644 --- a/tests/core/test_information_need.py +++ b/tests/core/test_information_need.py @@ -2,7 +2,7 @@ import pytest -from usersimcrs.core.information_need import ( +from usersimcrs.information_need_management.information_need import ( InformationNeed, generate_random_information_need, ) diff --git a/tests/simulator/agenda_based/test_interaction_model.py b/tests/simulator/agenda_based/test_interaction_model.py index 57516c28..8aabc08f 100644 --- a/tests/simulator/agenda_based/test_interaction_model.py +++ b/tests/simulator/agenda_based/test_interaction_model.py @@ -6,7 +6,12 @@ from dialoguekit.core.intent import Intent from dialoguekit.core.slot_value_annotation import SlotValueAnnotation from dialoguekit.utils.dialogue_reader import json_to_dialogues -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) +from usersimcrs.information_need_management.information_need_trackers import ( + HeuristicInformationNeedTracker, +) from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.simulator.agenda_based.interaction_model import ( InteractionModel, @@ -29,7 +34,7 @@ def im_crsv1( domain, ANNOTATED_CONVERSATIONS, ) - im.initialize_agenda(information_need) + im.initialize_agenda(HeuristicInformationNeedTracker(information_need)) return im @@ -45,7 +50,9 @@ def test_initialize_with_error(domain: SimulationDomain) -> None: def test_initialize_agenda( im_crsv1: InteractionModel, information_need: InformationNeed ) -> None: - im_crsv1.initialize_agenda(information_need) + im_crsv1.initialize_agenda( + HeuristicInformationNeedTracker(information_need) + ) assert len(im_crsv1.agenda.stack) == 6 assert im_crsv1.agenda.stack[0].intent == im_crsv1.INTENT_START assert im_crsv1.agenda.stack[1].intent == im_crsv1.INTENT_DISCLOSE diff --git a/tests/simulator/llm/test_stop_prompt.py b/tests/simulator/llm/test_stop_prompt.py index acced8e8..132979a1 100644 --- a/tests/simulator/llm/test_stop_prompt.py +++ b/tests/simulator/llm/test_stop_prompt.py @@ -2,7 +2,9 @@ import pytest -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.simulator.llm.prompt.stop_prompt import ( DEFAULT_STOP_DEFINITION, StopPrompt, diff --git a/tests/simulator/llm/test_utterance_generation_prompt.py b/tests/simulator/llm/test_utterance_generation_prompt.py index f1378df8..8a2e7ce0 100644 --- a/tests/simulator/llm/test_utterance_generation_prompt.py +++ b/tests/simulator/llm/test_utterance_generation_prompt.py @@ -4,7 +4,9 @@ from dialoguekit.core import Utterance from dialoguekit.participant import DialogueParticipant -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.simulator.llm.prompt.utterance_generation_prompt import ( DEFAULT_TASK_DEFINITION, UtteranceGenerationPrompt, diff --git a/usersimcrs/core/__init__.py b/usersimcrs/core/__init__.py index f7c76eed..fa93a90a 100644 --- a/usersimcrs/core/__init__.py +++ b/usersimcrs/core/__init__.py @@ -1,9 +1,7 @@ """Module level init for the core classes.""" -from usersimcrs.core.information_need import InformationNeed from usersimcrs.core.simulation_domain import SimulationDomain __all__ = [ - "InformationNeed", "SimulationDomain", ] diff --git a/usersimcrs/information_need_management/__init__.py b/usersimcrs/information_need_management/__init__.py new file mode 100644 index 00000000..50d49249 --- /dev/null +++ b/usersimcrs/information_need_management/__init__.py @@ -0,0 +1,19 @@ +"""Information need management.""" + +from usersimcrs.information_need_management.information_need import ( + InformationNeed, + generate_preference_grounded_information_need, + generate_random_information_need, +) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, + SlotUpdate, +) + +__all__ = [ + "InformationNeed", + "InformationNeedTracker", + "SlotUpdate", + "generate_preference_grounded_information_need", + "generate_random_information_need", +] diff --git a/usersimcrs/core/information_need.py b/usersimcrs/information_need_management/information_need.py similarity index 100% rename from usersimcrs/core/information_need.py rename to usersimcrs/information_need_management/information_need.py diff --git a/usersimcrs/simulator/information_need/information_need_tracker.py b/usersimcrs/information_need_management/information_need_tracker.py similarity index 75% rename from usersimcrs/simulator/information_need/information_need_tracker.py rename to usersimcrs/information_need_management/information_need_tracker.py index 0745df80..af4f24e2 100644 --- a/usersimcrs/simulator/information_need/information_need_tracker.py +++ b/usersimcrs/information_need_management/information_need_tracker.py @@ -14,7 +14,9 @@ from dialoguekit.core.dialogue_act import DialogueAct from dialoguekit.core.utterance import Utterance -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) @dataclass @@ -30,7 +32,15 @@ class InformationNeedTracker(ABC): def __init__(self, information_need: InformationNeed) -> None: """Initializes the tracker with the information need to update.""" - self.information_need = information_need + self._information_need = information_need + + def get_information_need(self) -> InformationNeed: + """Returns the tracked information need. + + Returns: + Information need. + """ + return self._information_need @abstractmethod def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: @@ -58,24 +68,19 @@ def update_from_agent_dialogue_acts( """ raise NotImplementedError - def apply_updates(self, updates: List[SlotUpdate]) -> None: - """Applies slot updates to the tracked information need.""" - self.apply_updates_to_information_need(self.information_need, updates) - - @staticmethod def apply_updates_to_information_need( - information_need: InformationNeed, updates: List[SlotUpdate] + self, updates: List[SlotUpdate] ) -> None: - """Applies slot updates to an information need.""" + """Applies slot updates to the tracked information need.""" for update in updates: if update.kind == "request": if update.status == InformationNeed.SLOT_STATE_COMPLETE: - information_need.mark_request_complete( + self._information_need.mark_request_complete( update.slot, update.value ) elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: - information_need.mark_request_attempted(update.slot) + self._information_need.mark_request_attempted(update.slot) elif update.status == InformationNeed.SLOT_STATE_COMPLETE: - information_need.mark_constraint_complete(update.slot) + self._information_need.mark_constraint_complete(update.slot) elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: - information_need.mark_constraint_attempted(update.slot) + self._information_need.mark_constraint_attempted(update.slot) diff --git a/usersimcrs/simulator/information_need/__init__.py b/usersimcrs/information_need_management/information_need_trackers/__init__.py similarity index 81% rename from usersimcrs/simulator/information_need/__init__.py rename to usersimcrs/information_need_management/information_need_trackers/__init__.py index 68fec1ab..5847e1d6 100644 --- a/usersimcrs/simulator/information_need/__init__.py +++ b/usersimcrs/information_need_management/information_need_trackers/__init__.py @@ -1,7 +1,7 @@ """Information need trackers.""" from .heuristic_interface import HeuristicInformationNeedTracker -from .information_need_tracker import ( +from usersimcrs.information_need_management.information_need_tracker import ( InformationNeedTracker, SlotUpdate, ) diff --git a/usersimcrs/simulator/information_need/heuristic_interface.py b/usersimcrs/information_need_management/information_need_trackers/heuristic_interface.py similarity index 84% rename from usersimcrs/simulator/information_need/heuristic_interface.py rename to usersimcrs/information_need_management/information_need_trackers/heuristic_interface.py index 22e676eb..e643f8f6 100644 --- a/usersimcrs/simulator/information_need/heuristic_interface.py +++ b/usersimcrs/information_need_management/information_need_trackers/heuristic_interface.py @@ -8,8 +8,10 @@ from dialoguekit.core.dialogue_act import DialogueAct from dialoguekit.core.utterance import Utterance -from usersimcrs.core.information_need import InformationNeed -from usersimcrs.simulator.information_need.information_need_tracker import ( +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) +from usersimcrs.information_need_management.information_need_tracker import ( InformationNeedTracker, SlotUpdate, ) @@ -29,7 +31,8 @@ def _get_target_slot_values(self, slot: str) -> List[str]: Normalized target values. """ normalized_values: List[str] = [] - for item in self.information_need.target_items: + information_need = self.get_information_need() + for item in information_need.target_items: value = item.get_property(slot) if value is None: continue @@ -73,7 +76,8 @@ def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: updates: List[SlotUpdate] = [] - for slot in self.information_need.request_states: + information_need = self.get_information_need() + for slot in information_need.request_states: normalized_slot = self._normalize_text(slot) target_values = self._get_target_slot_values(slot) value_mentioned = any(value in text for value in target_values) @@ -88,7 +92,7 @@ def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: ) ) - for slot, value in self.information_need.constraints.items(): + for slot, value in information_need.constraints.items(): normalized_slot = self._normalize_text(slot) values = value if isinstance(value, list) else [value] normalized_values = [self._normalize_text(str(v)) for v in values] @@ -115,20 +119,21 @@ def update_from_agent_dialogue_acts( for annotation in dialogue_act.annotations: slot = annotation.slot value = annotation.value - if slot in self.information_need.request_states: + information_need = self.get_information_need() + if slot in information_need.request_states: updates.append( self._slot_update( "request", slot, value is not None, value ) ) - if slot in self.information_need.constraint_states: + if slot in information_need.constraint_states: updates.append( self._slot_update( "constraint", slot, value - == self.information_need.get_constraint_value(slot), + == information_need.get_constraint_value(slot), ), ) return updates diff --git a/usersimcrs/simulator/information_need/llm_interface.py b/usersimcrs/information_need_management/information_need_trackers/llm_interface.py similarity index 87% rename from usersimcrs/simulator/information_need/llm_interface.py rename to usersimcrs/information_need_management/information_need_trackers/llm_interface.py index ded539a4..0da8aaff 100644 --- a/usersimcrs/simulator/information_need/llm_interface.py +++ b/usersimcrs/information_need_management/information_need_trackers/llm_interface.py @@ -5,7 +5,7 @@ from dialoguekit.core.dialogue_act import DialogueAct from dialoguekit.core.utterance import Utterance -from usersimcrs.simulator.information_need.information_need_tracker import ( +from usersimcrs.information_need_management.information_need_tracker import ( InformationNeedTracker, SlotUpdate, ) diff --git a/usersimcrs/simulator/agenda_based/agenda.py b/usersimcrs/simulator/agenda_based/agenda.py index 6a366ca8..5700daf4 100644 --- a/usersimcrs/simulator/agenda_based/agenda.py +++ b/usersimcrs/simulator/agenda_based/agenda.py @@ -12,7 +12,9 @@ from dialoguekit.core.dialogue_act import DialogueAct from dialoguekit.core.intent import Intent from dialoguekit.core.slot_value_annotation import SlotValueAnnotation -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) class Agenda: diff --git a/usersimcrs/simulator/agenda_based/agenda_based_simulator.py b/usersimcrs/simulator/agenda_based/agenda_based_simulator.py index e4659b58..c9f5ecd0 100644 --- a/usersimcrs/simulator/agenda_based/agenda_based_simulator.py +++ b/usersimcrs/simulator/agenda_based/agenda_based_simulator.py @@ -44,7 +44,7 @@ def __init__( super().__init__(id=id, domain=domain, item_collection=item_collection) self._preference_model = preference_model self._interaction_model = interaction_model - self._interaction_model.initialize_agenda(self.information_need) + self._interaction_model.initialize_agenda(self.information_need_tracker) self._nlu = nlu self._nlg = nlg self._dialogue_state_tracker = DialogueStateTracker() @@ -92,7 +92,7 @@ def generate_response( # Update agenda based on the agent's dialogue acts. self._interaction_model.update_agenda( - self.information_need, + self.information_need_tracker, self._preference_model, self._item_collection, ) diff --git a/usersimcrs/simulator/agenda_based/interaction_model.py b/usersimcrs/simulator/agenda_based/interaction_model.py index 3ef58a90..605a4556 100644 --- a/usersimcrs/simulator/agenda_based/interaction_model.py +++ b/usersimcrs/simulator/agenda_based/interaction_model.py @@ -20,16 +20,18 @@ from dialoguekit.core.intent import Intent from dialoguekit.core.slot_value_annotation import SlotValueAnnotation from dialoguekit.participant import DialogueParticipant -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.dialogue_management.dialogue_state_tracker import ( DialogueStateTracker, ) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.simulator.agenda_based.agenda import Agenda -from usersimcrs.simulator.information_need import ( - HeuristicInformationNeedTracker, -) from usersimcrs.user_modeling.preference_model import PreferenceModel _LEMMATIZER = WordNetLemmatizer() @@ -212,14 +214,16 @@ def _get_agent_user_interactions( user_dialogue_acts = [] return agent_user_interactions - def initialize_agenda(self, information_need: InformationNeed): + def initialize_agenda( + self, information_need_tracker: InformationNeedTracker + ): """Initializes user agenda. Args: - information_need: Information need. + information_need_tracker: Information need tracker. """ self.agenda = Agenda( - information_need, + information_need_tracker.get_information_need(), self.INTENT_DISCLOSE, # type: ignore[attr-defined] self.INTENT_INQUIRE, # type: ignore[attr-defined] self.INTENT_STOP, # type: ignore[attr-defined] @@ -303,7 +307,7 @@ def get_next_dialogue_acts(self, n: int = 1) -> List[DialogueAct]: def update_agenda( self, - information_need: InformationNeed, + information_need_tracker: InformationNeedTracker, preference_model: PreferenceModel, item_collection: ItemCollection, ) -> None: @@ -314,17 +318,15 @@ def update_agenda( neither. Once the push operations are done, we clean the stack. Args: - information_need: Information need. + information_need_tracker: Information need tracker. """ + information_need = information_need_tracker.get_information_need() current_state = self.dialogue_state_tracker.get_current_state() agent_dialogue_acts = current_state.agent_dialogue_acts[-1] - information_need_tracker = HeuristicInformationNeedTracker( - information_need - ) - information_need_tracker.apply_updates( + information_need_tracker.apply_updates_to_information_need( information_need_tracker.update_from_agent_dialogue_acts( agent_dialogue_acts - ) + ), ) user_dialogue_acts = [] for dialogue_act in agent_dialogue_acts: diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index abf7d9ec..6e396260 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -10,9 +10,6 @@ from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface -from usersimcrs.simulator.information_need import ( - HeuristicInformationNeedTracker, -) from usersimcrs.simulator.llm.prompt.stop_prompt import ( DEFAULT_STOP_DEFINITION, StopPrompt, @@ -50,15 +47,18 @@ def __init__( persona: Persona of the user. Defaults to None. """ super().__init__(id, domain, item_collection) - self._information_need_tracker = HeuristicInformationNeedTracker( - self.information_need - ) self.llm_interface = llm_interface self.generation_prompt = UtteranceGenerationPrompt( - self.information_need, item_type, task_definition, persona + self.information_need_tracker.get_information_need(), + item_type, + task_definition, + persona, ) self.stop_prompt = StopPrompt( - self.information_need, item_type, stop_definition, persona + self.information_need_tracker.get_information_need(), + item_type, + stop_definition, + persona, ) def _generate_response(self, agent_utterance: Utterance) -> Utterance: @@ -70,10 +70,10 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._information_need_tracker.apply_updates( - self._information_need_tracker.update_from_utterance( + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance( agent_utterance - ) + ), ) self.generation_prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index 51529a6f..36715f9b 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -9,9 +9,6 @@ from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface -from usersimcrs.simulator.information_need import ( - HeuristicInformationNeedTracker, -) from usersimcrs.simulator.llm.prompt.utterance_generation_prompt import ( DEFAULT_TASK_DEFINITION, UtteranceGenerationPrompt, @@ -42,12 +39,12 @@ def __init__( persona: Persona of the user. Defaults to None. """ super().__init__(id, domain, item_collection) - self._information_need_tracker = HeuristicInformationNeedTracker( - self.information_need - ) self.llm_interface = llm_interface self.prompt = UtteranceGenerationPrompt( - self.information_need, item_type, task_definition, persona + self.information_need_tracker.get_information_need(), + item_type, + task_definition, + persona, ) def _generate_response(self, agent_utterance: Utterance) -> Utterance: @@ -59,10 +56,10 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._information_need_tracker.apply_updates( - self._information_need_tracker.update_from_utterance( + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance( agent_utterance - ) + ), ) self.prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT diff --git a/usersimcrs/simulator/llm/prompt/prompt.py b/usersimcrs/simulator/llm/prompt/prompt.py index 619130ab..df24c990 100644 --- a/usersimcrs/simulator/llm/prompt/prompt.py +++ b/usersimcrs/simulator/llm/prompt/prompt.py @@ -4,7 +4,9 @@ from dialoguekit.core.utterance import Utterance from dialoguekit.participant.participant import DialogueParticipant -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.user_modeling.persona import Persona diff --git a/usersimcrs/simulator/llm/prompt/stop_prompt.py b/usersimcrs/simulator/llm/prompt/stop_prompt.py index c776e27b..f29ebb48 100644 --- a/usersimcrs/simulator/llm/prompt/stop_prompt.py +++ b/usersimcrs/simulator/llm/prompt/stop_prompt.py @@ -1,6 +1,8 @@ """Define the prompt for stopping the conversation.""" -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.simulator.llm.prompt.prompt import Prompt from usersimcrs.user_modeling.persona import Persona diff --git a/usersimcrs/simulator/llm/prompt/utterance_generation_prompt.py b/usersimcrs/simulator/llm/prompt/utterance_generation_prompt.py index 6f65a4e6..c18e2e2c 100644 --- a/usersimcrs/simulator/llm/prompt/utterance_generation_prompt.py +++ b/usersimcrs/simulator/llm/prompt/utterance_generation_prompt.py @@ -11,7 +11,9 @@ from dialoguekit.core.utterance import Utterance from dialoguekit.participant.participant import DialogueParticipant -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.simulator.llm.prompt.prompt import Prompt from usersimcrs.user_modeling.persona import Persona diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index af238c4e..f4e29c56 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -5,7 +5,12 @@ from dialoguekit.core.annotated_utterance import AnnotatedUtterance from dialoguekit.core.utterance import Utterance from dialoguekit.participant.user import User, UserType -from usersimcrs.core.information_need import generate_random_information_need +from usersimcrs.information_need_management.information_need import ( + generate_random_information_need, +) +from usersimcrs.information_need_management.information_need_trackers import ( + HeuristicInformationNeedTracker, +) from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection @@ -25,8 +30,10 @@ def __init__( def get_new_information_need(self) -> None: """Generates a new information need.""" - self.information_need = generate_random_information_need( - self._domain, self._item_collection + self.information_need_tracker = HeuristicInformationNeedTracker( + generate_random_information_need( + self._domain, self._item_collection + ) ) @abstractmethod From a0c7fcb438a0b6076b7bf46bf406071dcfce383b Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 21 Jul 2026 14:10:37 +0200 Subject: [PATCH 17/22] add a comment --- .../information_need_tracker.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/usersimcrs/information_need_management/information_need_tracker.py b/usersimcrs/information_need_management/information_need_tracker.py index af4f24e2..23fa6ff4 100644 --- a/usersimcrs/information_need_management/information_need_tracker.py +++ b/usersimcrs/information_need_management/information_need_tracker.py @@ -28,8 +28,6 @@ class SlotUpdate: class InformationNeedTracker(ABC): - """Tracks conversation progress against an information need.""" - def __init__(self, information_need: InformationNeed) -> None: """Initializes the tracker with the information need to update.""" self._information_need = information_need @@ -71,7 +69,11 @@ def update_from_agent_dialogue_acts( def apply_updates_to_information_need( self, updates: List[SlotUpdate] ) -> None: - """Applies slot updates to the tracked information need.""" + """Applies slot updates to the tracked information need. + + Args: + updates: Slot updates to apply to the tracked information need. + """ for update in updates: if update.kind == "request": if update.status == InformationNeed.SLOT_STATE_COMPLETE: From ae91d71deb568d554029e60cc8f111b90ddb1100 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 4 Aug 2026 16:11:16 +0200 Subject: [PATCH 18/22] resolve comments --- tests/conftest.py | 2 +- tests/core/test_information_need.py | 17 +--- .../information_need.py | 11 +-- .../information_need_tracker.py | 18 +++- .../information_need_trackers/__init__.py | 10 +- ... => heuristic_information_need_tracker.py} | 39 +++++++- .../llm_information_need_tracker.py | 91 +++++++++++++++++++ .../llm_interface.py | 22 ----- .../llm/llm_dual_prompt_user_simulator.py | 18 ++-- .../llm/llm_single_prompt_user_simulator.py | 15 +-- usersimcrs/simulator/user_simulator.py | 38 +++++++- 11 files changed, 211 insertions(+), 70 deletions(-) rename usersimcrs/information_need_management/information_need_trackers/{heuristic_interface.py => heuristic_information_need_tracker.py} (82%) create mode 100644 usersimcrs/information_need_management/information_need_trackers/llm_information_need_tracker.py delete mode 100644 usersimcrs/information_need_management/information_need_trackers/llm_interface.py diff --git a/tests/conftest.py b/tests/conftest.py index 0fd05314..368f0f3b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,7 @@ def domain() -> SimulationDomain: return SimulationDomain(DOMAIN_YAML_FILE) -@pytest.fixture(scope="module") +@pytest.fixture def information_need() -> InformationNeed: """Information need fixture.""" constraints = {"GENRE": "Comedy", "DIRECTOR": "Steven Spielberg"} diff --git a/tests/core/test_information_need.py b/tests/core/test_information_need.py index db4b5fbb..889e1268 100644 --- a/tests/core/test_information_need.py +++ b/tests/core/test_information_need.py @@ -63,23 +63,8 @@ def test_get_requestable_slots(information_need: InformationNeed) -> None: assert information_need.get_requestable_slots() == ["PLOT"] -def test_to_dict() -> None: +def test_to_dict(information_need: InformationNeed) -> None: """Test to_dict.""" - information_need = InformationNeed( - [ - Item( - "1", - { - "GENRE": "Comedy", - "DIRECTOR": "Steven Spielberg", - "RATING": 4.5, - "PLOT": "A movie plot", - }, - ) - ], - {"GENRE": "Comedy", "DIRECTOR": "Steven Spielberg"}, - ["PLOT", "RATING"], - ) assert information_need.to_dict() == { "target_items": [ { diff --git a/usersimcrs/information_need_management/information_need.py b/usersimcrs/information_need_management/information_need.py index 919e6f96..4f0fdf34 100644 --- a/usersimcrs/information_need_management/information_need.py +++ b/usersimcrs/information_need_management/information_need.py @@ -151,6 +151,9 @@ def __init__( constraint_states: Optional states for constraints. request_states: Optional states for requests. """ + constraint_states = constraint_states or {} + request_states = request_states or {} + self.target_items = target_items self.constraints = constraints self.requested_slots = defaultdict( @@ -159,9 +162,7 @@ def __init__( self.constraint_states: DefaultDict[str, str] = defaultdict( lambda: self.SLOT_STATE_INCOMPLETE, { - slot: (constraint_states or {}).get( - slot, self.SLOT_STATE_INCOMPLETE - ) + slot: constraint_states.get(slot, self.SLOT_STATE_INCOMPLETE) for slot in constraints }, ) @@ -169,9 +170,7 @@ def __init__( self.request_states: DefaultDict[str, str] = defaultdict( lambda: self.SLOT_STATE_INCOMPLETE, { - slot: (request_states or {}).get( - slot, self.SLOT_STATE_INCOMPLETE - ) + slot: request_states.get(slot, self.SLOT_STATE_INCOMPLETE) for slot in requests }, ) diff --git a/usersimcrs/information_need_management/information_need_tracker.py b/usersimcrs/information_need_management/information_need_tracker.py index 23fa6ff4..b40cc225 100644 --- a/usersimcrs/information_need_management/information_need_tracker.py +++ b/usersimcrs/information_need_management/information_need_tracker.py @@ -32,6 +32,14 @@ def __init__(self, information_need: InformationNeed) -> None: """Initializes the tracker with the information need to update.""" self._information_need = information_need + def set_information_need(self, information_need: InformationNeed) -> None: + """Sets the information need tracked by this tracker. + + Args: + information_need: Information need to track. + """ + self._information_need = information_need + def get_information_need(self) -> InformationNeed: """Returns the tracked information need. @@ -47,8 +55,11 @@ def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: Args: utterance: Utterance to inspect. - Return: + Returns: List of slot updates. + + Raises: + NotImplementedError: If not implemented in derived class. """ raise NotImplementedError @@ -61,8 +72,11 @@ def update_from_agent_dialogue_acts( Args: agent_dialogue_acts: Agent dialogue acts to inspect. - Return: + Returns: List of slot updates. + + Raises: + NotImplementedError: If not implemented in derived class. """ raise NotImplementedError diff --git a/usersimcrs/information_need_management/information_need_trackers/__init__.py b/usersimcrs/information_need_management/information_need_trackers/__init__.py index 5847e1d6..c7cdd423 100644 --- a/usersimcrs/information_need_management/information_need_trackers/__init__.py +++ b/usersimcrs/information_need_management/information_need_trackers/__init__.py @@ -1,13 +1,19 @@ """Information need trackers.""" -from .heuristic_interface import HeuristicInformationNeedTracker +from .heuristic_information_need_tracker import ( + HeuristicInformationNeedTracker, +) from usersimcrs.information_need_management.information_need_tracker import ( InformationNeedTracker, SlotUpdate, ) -from .llm_interface import LLMInformationNeedTracker +from .llm_information_need_tracker import ( + DEFAULT_UPDATE_PROMPT, + LLMInformationNeedTracker, +) __all__ = [ + "DEFAULT_UPDATE_PROMPT", "HeuristicInformationNeedTracker", "InformationNeedTracker", "LLMInformationNeedTracker", diff --git a/usersimcrs/information_need_management/information_need_trackers/heuristic_interface.py b/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py similarity index 82% rename from usersimcrs/information_need_management/information_need_trackers/heuristic_interface.py rename to usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py index e643f8f6..837ff6e8 100644 --- a/usersimcrs/information_need_management/information_need_trackers/heuristic_interface.py +++ b/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py @@ -1,9 +1,14 @@ -"""Heuristic information need tracker.""" +"""Heuristic information need tracker. + +The tracker inspects utterance text and dialogue-act annotations for mentions of +the slots and values contained in an information need. Slot mentions mark the +slot as attempted, while matching target or constraint values mark it complete. +""" from __future__ import annotations import re -from typing import List +from typing import Any, List from dialoguekit.core.dialogue_act import DialogueAct from dialoguekit.core.utterance import Utterance @@ -19,6 +24,14 @@ class HeuristicInformationNeedTracker(InformationNeedTracker): def _normalize_text(self, text: str) -> str: + """Normalizes text for matching. + + Args: + text: Text to normalize. + + Returns: + Normalized text. + """ return " ".join(re.sub(r"[_-]", " ", text.lower()).split()) def _get_target_slot_values(self, slot: str) -> List[str]: @@ -27,7 +40,7 @@ def _get_target_slot_values(self, slot: str) -> List[str]: Args: slot: Slot name. - Return: + Returns: Normalized target values. """ normalized_values: List[str] = [] @@ -43,7 +56,7 @@ def _get_target_slot_values(self, slot: str) -> List[str]: return normalized_values def _slot_update( - self, kind: str, slot: str, is_complete: bool, value=None + self, kind: str, slot: str, is_complete: bool, value: Any = None ) -> SlotUpdate: """Builds a slot update with complete or attempted status. @@ -53,7 +66,7 @@ def _slot_update( is_complete: Whether the slot is complete. value: Slot value for request completion. - Return: + Returns: Slot update. """ return SlotUpdate( @@ -68,6 +81,14 @@ def _slot_update( ) def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: + """Builds information-need updates from an utterance. + + Args: + utterance: Utterance to inspect. + + Returns: + List of slot updates. + """ raw = getattr(utterance, "text", "") text = self._normalize_text(raw) @@ -114,6 +135,14 @@ def update_from_agent_dialogue_acts( self, agent_dialogue_acts: List[DialogueAct], ) -> List[SlotUpdate]: + """Builds information-need updates from agent dialogue acts. + + Args: + agent_dialogue_acts: Agent dialogue acts to inspect. + + Returns: + List of slot updates. + """ updates: List[SlotUpdate] = [] for dialogue_act in agent_dialogue_acts: for annotation in dialogue_act.annotations: diff --git a/usersimcrs/information_need_management/information_need_trackers/llm_information_need_tracker.py b/usersimcrs/information_need_management/information_need_trackers/llm_information_need_tracker.py new file mode 100644 index 00000000..cf03a4d8 --- /dev/null +++ b/usersimcrs/information_need_management/information_need_trackers/llm_information_need_tracker.py @@ -0,0 +1,91 @@ +"""LLM-backed information need tracker. + +This module defines the prompt shape and class contract for an LLM-backed +tracker. The tracking logic is intentionally left unimplemented until the prompt +format and output parsing contract are finalized. +""" + +from __future__ import annotations + +from typing import List + +from dialoguekit.core.dialogue_act import DialogueAct +from dialoguekit.core.utterance import Utterance + +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, + SlotUpdate, +) +from usersimcrs.llm_interfaces.llm_interface import LLMInterface + +DEFAULT_UPDATE_PROMPT = """\ +You update the state of a simulated user's information need. + +Information need: +{information_need} + +Conversation evidence: +{evidence} + +Return only JSON as a list of objects with keys: +- kind: "constraint" or "request" +- slot: the slot name +- status: "attempted" or "complete" +- value: the observed value, or null +""" + + +class LLMInformationNeedTracker(InformationNeedTracker): + def __init__( + self, + information_need: InformationNeed, + llm_interface: LLMInterface, + prompt_template: str = DEFAULT_UPDATE_PROMPT, + ) -> None: + """Initializes the LLM-backed tracker. + + Args: + information_need: Information need to track. + llm_interface: Interface to the large language model. + prompt_template: Prompt template containing information_need and + evidence placeholders. + """ + super().__init__(information_need) + self.llm_interface = llm_interface + self.prompt_template = prompt_template + + def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: + """Builds information-need updates from an utterance. + + Args: + utterance: Utterance to inspect. + + Returns: + List of slot updates. + + Raises: + NotImplementedError: Until the LLM tracking implementation is + defined. + """ + raise NotImplementedError + + def update_from_agent_dialogue_acts( + self, + agent_dialogue_acts: List[DialogueAct], + ) -> List[SlotUpdate]: + """Builds information-need updates from agent dialogue acts. + + Args: + agent_dialogue_acts: Agent dialogue acts to inspect. + + Returns: + List of slot updates. + + Raises: + NotImplementedError: Until the LLM tracking implementation is + defined. + """ + raise NotImplementedError diff --git a/usersimcrs/information_need_management/information_need_trackers/llm_interface.py b/usersimcrs/information_need_management/information_need_trackers/llm_interface.py deleted file mode 100644 index 0da8aaff..00000000 --- a/usersimcrs/information_need_management/information_need_trackers/llm_interface.py +++ /dev/null @@ -1,22 +0,0 @@ -"""LLM-backed information need tracker.""" - -from __future__ import annotations - -from dialoguekit.core.dialogue_act import DialogueAct -from dialoguekit.core.utterance import Utterance - -from usersimcrs.information_need_management.information_need_tracker import ( - InformationNeedTracker, - SlotUpdate, -) - - -class LLMInformationNeedTracker(InformationNeedTracker): - def update_from_utterance(self, utterance: Utterance) -> list[SlotUpdate]: - raise NotImplementedError - - def update_from_agent_dialogue_acts( - self, - agent_dialogue_acts: list[DialogueAct], - ) -> list[SlotUpdate]: - raise NotImplementedError diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index 6e396260..2bd9c593 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -8,6 +8,9 @@ from dialoguekit.core.utterance import Utterance from dialoguekit.participant import DialogueParticipant from usersimcrs.core.simulation_domain import SimulationDomain +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface from usersimcrs.simulator.llm.prompt.stop_prompt import ( @@ -33,6 +36,7 @@ def __init__( task_definition: str = DEFAULT_TASK_DEFINITION, stop_definition: str = DEFAULT_STOP_DEFINITION, persona: Persona = None, + information_need_tracker: InformationNeedTracker = None, ) -> None: """Initializes the user simulator. @@ -45,8 +49,10 @@ def __init__( stop_definition: Definition of the stop task. Defaults to DEFAULT_STOP_DEFINITION. persona: Persona of the user. Defaults to None. + information_need_tracker: Tracker to use for the generated + information need. Defaults to a heuristic tracker. """ - super().__init__(id, domain, item_collection) + super().__init__(id, domain, item_collection, information_need_tracker) self.llm_interface = llm_interface self.generation_prompt = UtteranceGenerationPrompt( self.information_need_tracker.get_information_need(), @@ -70,11 +76,7 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self.information_need_tracker.apply_updates_to_information_need( - self.information_need_tracker.update_from_utterance( - agent_utterance - ), - ) + self._update_information_need_from_utterance(agent_utterance) self.generation_prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) @@ -95,4 +97,8 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: self.generation_prompt.update_prompt_context( user_utterance, DialogueParticipant.USER ) + self.stop_prompt.update_prompt_context( + user_utterance, DialogueParticipant.USER + ) + self._update_information_need_from_utterance(user_utterance) return user_utterance diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index 36715f9b..69510b49 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -7,6 +7,9 @@ from dialoguekit.core.utterance import Utterance from dialoguekit.participant import DialogueParticipant from usersimcrs.core.simulation_domain import SimulationDomain +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface from usersimcrs.simulator.llm.prompt.utterance_generation_prompt import ( @@ -27,6 +30,7 @@ def __init__( item_type: str, task_definition: str = DEFAULT_TASK_DEFINITION, persona: Persona = None, + information_need_tracker: InformationNeedTracker = None, ) -> None: """Initializes the user simulator. @@ -37,8 +41,10 @@ def __init__( task_definition: Definition of the task to be performed. Defaults to DEFAULT_TASK_DEFINITION. persona: Persona of the user. Defaults to None. + information_need_tracker: Tracker to use for the generated + information need. Defaults to a heuristic tracker. """ - super().__init__(id, domain, item_collection) + super().__init__(id, domain, item_collection, information_need_tracker) self.llm_interface = llm_interface self.prompt = UtteranceGenerationPrompt( self.information_need_tracker.get_information_need(), @@ -56,11 +62,7 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self.information_need_tracker.apply_updates_to_information_need( - self.information_need_tracker.update_from_utterance( - agent_utterance - ), - ) + self._update_information_need_from_utterance(agent_utterance) self.prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) @@ -68,4 +70,5 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: self.prompt.update_prompt_context( user_utterance, DialogueParticipant.USER ) + self._update_information_need_from_utterance(user_utterance) return user_utterance diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index f4e29c56..66597fb8 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -1,6 +1,7 @@ """User simulator abstract class.""" from abc import ABC, abstractmethod +from typing import Optional from dialoguekit.core.annotated_utterance import AnnotatedUtterance from dialoguekit.core.utterance import Utterance @@ -11,6 +12,9 @@ from usersimcrs.information_need_management.information_need_trackers import ( HeuristicInformationNeedTracker, ) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection @@ -21,19 +25,45 @@ def __init__( id: str, domain: SimulationDomain, item_collection: ItemCollection, + information_need_tracker: Optional[InformationNeedTracker] = None, ) -> None: - """Initializes the user simulator.""" + """Initializes the user simulator. + + Args: + id: Simulator ID. + domain: Domain. + item_collection: Item collection. + information_need_tracker: Tracker to use for the generated + information need. Defaults to a heuristic tracker. + """ super().__init__(id, UserType.SIMULATOR) self._domain = domain self._item_collection = item_collection + self.information_need_tracker = information_need_tracker self.get_new_information_need() def get_new_information_need(self) -> None: """Generates a new information need.""" - self.information_need_tracker = HeuristicInformationNeedTracker( - generate_random_information_need( - self._domain, self._item_collection + information_need = generate_random_information_need( + self._domain, self._item_collection + ) + if self.information_need_tracker is None: + self.information_need_tracker = HeuristicInformationNeedTracker( + information_need ) + else: + self.information_need_tracker.set_information_need(information_need) + + def _update_information_need_from_utterance( + self, utterance: Utterance + ) -> None: + """Updates the information need from an utterance. + + Args: + utterance: Utterance to inspect. + """ + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance(utterance), ) @abstractmethod From 440038ed308e10252941ae25d79b514de090965b Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 11 Aug 2026 10:27:42 +0200 Subject: [PATCH 19/22] fix information need --- tests/conftest.py | 2 +- .../information_need.py | 25 +++-- .../information_need_tracker.py | 23 ++--- .../information_need_trackers/__init__.py | 5 - .../heuristic_information_need_tracker.py | 7 +- .../llm_information_need_tracker.py | 91 ------------------- .../agenda_based/interaction_model.py | 9 +- .../llm/llm_dual_prompt_user_simulator.py | 8 +- .../llm/llm_single_prompt_user_simulator.py | 8 +- usersimcrs/simulator/user_simulator.py | 14 +-- 10 files changed, 50 insertions(+), 142 deletions(-) delete mode 100644 usersimcrs/information_need_management/information_need_trackers/llm_information_need_tracker.py diff --git a/tests/conftest.py b/tests/conftest.py index 368f0f3b..fc8147d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,7 @@ def information_need() -> InformationNeed: return InformationNeed(target_items, constraints, requests) -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def item_collection(domain: SimulationDomain): """Item collection fixture.""" item_collection = ItemCollection("tests/data/items.db", "test_collection") diff --git a/usersimcrs/information_need_management/information_need.py b/usersimcrs/information_need_management/information_need.py index 4f0fdf34..b743a6b9 100644 --- a/usersimcrs/information_need_management/information_need.py +++ b/usersimcrs/information_need_management/information_need.py @@ -11,7 +11,7 @@ import random from collections import defaultdict -from typing import Any, DefaultDict, Dict, List, Optional +from typing import Any, DefaultDict, Dict, List from dialoguekit.core.slot_value_annotation import SlotValueAnnotation from usersimcrs.core.simulation_domain import SimulationDomain @@ -63,8 +63,8 @@ def generate_preference_grounded_information_need( ) -> InformationNeed: """Generates an information need aligned with a preference model. - The function first samples preferred slot-value pairs from the preference - model, then tries to find an item matching them. If no matching item is + The function samples preferred slot-value pairs from the preference model, + then tries to find an item matching that subset. If no matching item is found, the information need is returned without target items. Args: @@ -82,6 +82,14 @@ def generate_preference_grounded_information_need( if value is not None } + if preferred_constraints: + preferred_constraints = dict( + random.sample( + list(preferred_constraints.items()), + random.randint(1, len(preferred_constraints)), + ), + ) + matching_items = item_collection.get_items_by_properties( [ SlotValueAnnotation(slot, value) @@ -138,8 +146,8 @@ def __init__( target_items: List[Item], constraints: Dict[str, Any], requests: List[str], - constraint_states: Optional[Dict[str, str]] = None, - request_states: Optional[Dict[str, str]] = None, + constraint_states: Dict[str, str] = {}, + request_states: Dict[str, str] = {}, ) -> None: """Initializes an information need. @@ -148,12 +156,9 @@ def __init__( constraints: Slot-value pairs representing constraints on the item of interest. requests: Slots representing the desired information. - constraint_states: Optional states for constraints. - request_states: Optional states for requests. + constraint_states: States for constraints. + request_states: States for requests. """ - constraint_states = constraint_states or {} - request_states = request_states or {} - self.target_items = target_items self.constraints = constraints self.requested_slots = defaultdict( diff --git a/usersimcrs/information_need_management/information_need_tracker.py b/usersimcrs/information_need_management/information_need_tracker.py index b40cc225..3a52ca2c 100644 --- a/usersimcrs/information_need_management/information_need_tracker.py +++ b/usersimcrs/information_need_management/information_need_tracker.py @@ -7,6 +7,7 @@ from __future__ import annotations +import copy from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, List @@ -30,7 +31,7 @@ class SlotUpdate: class InformationNeedTracker(ABC): def __init__(self, information_need: InformationNeed) -> None: """Initializes the tracker with the information need to update.""" - self._information_need = information_need + self.set_information_need(information_need) def set_information_need(self, information_need: InformationNeed) -> None: """Sets the information need tracked by this tracker. @@ -38,14 +39,10 @@ def set_information_need(self, information_need: InformationNeed) -> None: Args: information_need: Information need to track. """ - self._information_need = information_need + self._information_need = copy.deepcopy(information_need) def get_information_need(self) -> InformationNeed: - """Returns the tracked information need. - - Returns: - Information need. - """ + """Returns the tracked information need.""" return self._information_need @abstractmethod @@ -55,11 +52,11 @@ def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: Args: utterance: Utterance to inspect. - Returns: - List of slot updates. - Raises: NotImplementedError: If not implemented in derived class. + + Returns: + List of slot updates. """ raise NotImplementedError @@ -72,11 +69,11 @@ def update_from_agent_dialogue_acts( Args: agent_dialogue_acts: Agent dialogue acts to inspect. - Returns: - List of slot updates. - Raises: NotImplementedError: If not implemented in derived class. + + Returns: + List of slot updates. """ raise NotImplementedError diff --git a/usersimcrs/information_need_management/information_need_trackers/__init__.py b/usersimcrs/information_need_management/information_need_trackers/__init__.py index c7cdd423..bd312972 100644 --- a/usersimcrs/information_need_management/information_need_trackers/__init__.py +++ b/usersimcrs/information_need_management/information_need_trackers/__init__.py @@ -7,15 +7,10 @@ InformationNeedTracker, SlotUpdate, ) -from .llm_information_need_tracker import ( - DEFAULT_UPDATE_PROMPT, - LLMInformationNeedTracker, -) __all__ = [ "DEFAULT_UPDATE_PROMPT", "HeuristicInformationNeedTracker", "InformationNeedTracker", - "LLMInformationNeedTracker", "SlotUpdate", ] diff --git a/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py b/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py index 837ff6e8..a99d3cb9 100644 --- a/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py +++ b/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py @@ -104,12 +104,15 @@ def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: value_mentioned = any(value in text for value in target_values) slot_mentioned = normalized_slot in text if slot_mentioned or value_mentioned: + is_complete = value_mentioned or ( + slot_mentioned and not information_need.target_items + ) updates.append( self._slot_update( "request", slot, - value_mentioned, - raw.strip() if value_mentioned else None, + is_complete, + raw.strip() if is_complete else None, ) ) diff --git a/usersimcrs/information_need_management/information_need_trackers/llm_information_need_tracker.py b/usersimcrs/information_need_management/information_need_trackers/llm_information_need_tracker.py deleted file mode 100644 index cf03a4d8..00000000 --- a/usersimcrs/information_need_management/information_need_trackers/llm_information_need_tracker.py +++ /dev/null @@ -1,91 +0,0 @@ -"""LLM-backed information need tracker. - -This module defines the prompt shape and class contract for an LLM-backed -tracker. The tracking logic is intentionally left unimplemented until the prompt -format and output parsing contract are finalized. -""" - -from __future__ import annotations - -from typing import List - -from dialoguekit.core.dialogue_act import DialogueAct -from dialoguekit.core.utterance import Utterance - -from usersimcrs.information_need_management.information_need import ( - InformationNeed, -) -from usersimcrs.information_need_management.information_need_tracker import ( - InformationNeedTracker, - SlotUpdate, -) -from usersimcrs.llm_interfaces.llm_interface import LLMInterface - -DEFAULT_UPDATE_PROMPT = """\ -You update the state of a simulated user's information need. - -Information need: -{information_need} - -Conversation evidence: -{evidence} - -Return only JSON as a list of objects with keys: -- kind: "constraint" or "request" -- slot: the slot name -- status: "attempted" or "complete" -- value: the observed value, or null -""" - - -class LLMInformationNeedTracker(InformationNeedTracker): - def __init__( - self, - information_need: InformationNeed, - llm_interface: LLMInterface, - prompt_template: str = DEFAULT_UPDATE_PROMPT, - ) -> None: - """Initializes the LLM-backed tracker. - - Args: - information_need: Information need to track. - llm_interface: Interface to the large language model. - prompt_template: Prompt template containing information_need and - evidence placeholders. - """ - super().__init__(information_need) - self.llm_interface = llm_interface - self.prompt_template = prompt_template - - def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: - """Builds information-need updates from an utterance. - - Args: - utterance: Utterance to inspect. - - Returns: - List of slot updates. - - Raises: - NotImplementedError: Until the LLM tracking implementation is - defined. - """ - raise NotImplementedError - - def update_from_agent_dialogue_acts( - self, - agent_dialogue_acts: List[DialogueAct], - ) -> List[SlotUpdate]: - """Builds information-need updates from agent dialogue acts. - - Args: - agent_dialogue_acts: Agent dialogue acts to inspect. - - Returns: - List of slot updates. - - Raises: - NotImplementedError: Until the LLM tracking implementation is - defined. - """ - raise NotImplementedError diff --git a/usersimcrs/simulator/agenda_based/interaction_model.py b/usersimcrs/simulator/agenda_based/interaction_model.py index 605a4556..99c3c6c3 100644 --- a/usersimcrs/simulator/agenda_based/interaction_model.py +++ b/usersimcrs/simulator/agenda_based/interaction_model.py @@ -630,9 +630,12 @@ def _sample_next_user_dialogue_acts( ) ) elif sampled_intent == self.INTENT_INQUIRE: # type: ignore[attr-defined] # noqa - slot = random.choice(information_need.get_requestable_slots()) - if not slot: - slot = random.choice(self._domain.get_requestable_slots()) + requestable_slots = information_need.get_requestable_slots() + slot = ( + random.choice(requestable_slots) + if requestable_slots + else random.choice(self._domain.get_requestable_slots()) + ) if slot in information_need.request_states: information_need.mark_request_attempted(slot) user_dialogue_acts.append( diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index 2bd9c593..1ffcf942 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -76,7 +76,9 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._update_information_need_from_utterance(agent_utterance) + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance(agent_utterance) + ) self.generation_prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) @@ -100,5 +102,7 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: self.stop_prompt.update_prompt_context( user_utterance, DialogueParticipant.USER ) - self._update_information_need_from_utterance(user_utterance) + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance(user_utterance) + ) return user_utterance diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index 69510b49..343daf62 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -62,7 +62,9 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ - self._update_information_need_from_utterance(agent_utterance) + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance(agent_utterance) + ) self.prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) @@ -70,5 +72,7 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: self.prompt.update_prompt_context( user_utterance, DialogueParticipant.USER ) - self._update_information_need_from_utterance(user_utterance) + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance(user_utterance) + ) return user_utterance diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index 66597fb8..8c93e84e 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -34,7 +34,7 @@ def __init__( domain: Domain. item_collection: Item collection. information_need_tracker: Tracker to use for the generated - information need. Defaults to a heuristic tracker. + information need. Defaults to None. """ super().__init__(id, UserType.SIMULATOR) self._domain = domain @@ -54,18 +54,6 @@ def get_new_information_need(self) -> None: else: self.information_need_tracker.set_information_need(information_need) - def _update_information_need_from_utterance( - self, utterance: Utterance - ) -> None: - """Updates the information need from an utterance. - - Args: - utterance: Utterance to inspect. - """ - self.information_need_tracker.apply_updates_to_information_need( - self.information_need_tracker.update_from_utterance(utterance), - ) - @abstractmethod def _generate_response(self, agent_utterance: Utterance) -> Utterance: """Generates response to the agent utterance. From 8453319f26a0804e47d8f230912965ca8d254807 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 11 Aug 2026 16:04:28 +0200 Subject: [PATCH 20/22] resolve comments --- .../information_need.py | 64 +++++++++---------- .../heuristic_information_need_tracker.py | 20 ++++-- .../agenda_based/agenda_based_simulator.py | 12 +++- .../llm/llm_dual_prompt_user_simulator.py | 5 +- .../llm/llm_single_prompt_user_simulator.py | 5 +- usersimcrs/simulator/user_simulator.py | 25 +------- usersimcrs/utils/simulation_utils.py | 36 +++++++++++ 7 files changed, 99 insertions(+), 68 deletions(-) diff --git a/usersimcrs/information_need_management/information_need.py b/usersimcrs/information_need_management/information_need.py index b743a6b9..914e213b 100644 --- a/usersimcrs/information_need_management/information_need.py +++ b/usersimcrs/information_need_management/information_need.py @@ -63,9 +63,11 @@ def generate_preference_grounded_information_need( ) -> InformationNeed: """Generates an information need aligned with a preference model. - The function samples preferred slot-value pairs from the preference model, - then tries to find an item matching that subset. If no matching item is - found, the information need is returned without target items. + The function samples a random number of informable slots, obtains + preference constraints for those slots, then tries to find an item matching + them. If no matching item is found, constraints are relaxed until a match is + found. If no constraints match any items, the information need is returned + without target items. Args: domain: Domain knowledge. @@ -75,49 +77,45 @@ def generate_preference_grounded_information_need( Returns: Information need. """ + informable_slots = list(domain.get_informable_slots()) + preferred_slots = random.sample( + informable_slots, random.randint(1, len(informable_slots)) + ) preferred_constraints = { slot: value - for slot in domain.get_informable_slots() + for slot in preferred_slots for value, _ in [preference_model.get_slot_preference(slot)] if value is not None } - if preferred_constraints: - preferred_constraints = dict( - random.sample( - list(preferred_constraints.items()), - random.randint(1, len(preferred_constraints)), - ), + matching_items = [] + matching_constraints = preferred_constraints + preferred_constraint_items = list(preferred_constraints.items()) + for num_constraints in range(len(preferred_constraint_items), 0, -1): + candidate_constraints = dict( + preferred_constraint_items[:num_constraints] + ) + matching_items = item_collection.get_items_by_properties( + [ + SlotValueAnnotation(slot, value) + for slot, value in candidate_constraints.items() + ] ) + if matching_items: + matching_constraints = candidate_constraints + break - matching_items = item_collection.get_items_by_properties( - [ - SlotValueAnnotation(slot, value) - for slot, value in preferred_constraints.items() - ] - ) target_items = [random.choice(matching_items)] if matching_items else [] - constraint_source = ( - { - slot: target_items[0].get_property(slot) - for slot in domain.get_informable_slots() - if slot in target_items[0].properties - } - if target_items - else preferred_constraints - ) - constraints = ( { - slot: constraint_source[slot] - for slot in random.sample( - list(constraint_source), - random.randint(1, len(constraint_source)), - ) + slot: value + for slot in matching_constraints + for value in [target_items[0].get_property(slot)] + if value is not None } - if constraint_source - else {} + if target_items + else matching_constraints ) requestable_slots = [ diff --git a/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py b/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py index a99d3cb9..4012af9f 100644 --- a/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py +++ b/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py @@ -104,15 +104,25 @@ def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: value_mentioned = any(value in text for value in target_values) slot_mentioned = normalized_slot in text if slot_mentioned or value_mentioned: - is_complete = value_mentioned or ( - slot_mentioned and not information_need.target_items - ) updates.append( self._slot_update( "request", slot, - is_complete, - raw.strip() if is_complete else None, + value_mentioned, + raw.strip() if value_mentioned else None, + ) + ) + + if not information_need.target_items and not updates: + attempted_requests = [ + slot + for slot, state in information_need.request_states.items() + if state == InformationNeed.SLOT_STATE_ATTEMPTED + ] + if len(attempted_requests) == 1: + updates.append( + self._slot_update( + "request", attempted_requests[0], True, raw.strip() ) ) diff --git a/usersimcrs/simulator/agenda_based/agenda_based_simulator.py b/usersimcrs/simulator/agenda_based/agenda_based_simulator.py index c9f5ecd0..fdd3b604 100644 --- a/usersimcrs/simulator/agenda_based/agenda_based_simulator.py +++ b/usersimcrs/simulator/agenda_based/agenda_based_simulator.py @@ -10,6 +10,9 @@ from usersimcrs.dialogue_management.dialogue_state_tracker import ( DialogueStateTracker, ) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.items.ratings import Ratings from usersimcrs.simulator.agenda_based.interaction_model import InteractionModel @@ -25,6 +28,7 @@ def __init__( item_collection: ItemCollection, preference_model: PreferenceModel, interaction_model: InteractionModel, + information_need_tracker: InformationNeedTracker, nlu: NLU, nlg: ConditionalNLG, ratings: Ratings, @@ -37,11 +41,17 @@ def __init__( item_collection: Item collection. preference_model: Preference model. interaction_model: Interaction model. + information_need_tracker: Tracker for the information need. nlu: NLU module performing dialogue act extraction. nlg: NLG module generating textual responses. ratings: Historical ratings. """ - super().__init__(id=id, domain=domain, item_collection=item_collection) + super().__init__( + id=id, + domain=domain, + item_collection=item_collection, + information_need_tracker=information_need_tracker, + ) self._preference_model = preference_model self._interaction_model = interaction_model self._interaction_model.initialize_agenda(self.information_need_tracker) diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index 1ffcf942..de6d9be6 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -33,10 +33,10 @@ def __init__( item_collection: ItemCollection, llm_interface: LLMInterface, item_type: str, + information_need_tracker: InformationNeedTracker, task_definition: str = DEFAULT_TASK_DEFINITION, stop_definition: str = DEFAULT_STOP_DEFINITION, persona: Persona = None, - information_need_tracker: InformationNeedTracker = None, ) -> None: """Initializes the user simulator. @@ -44,13 +44,12 @@ def __init__( id: User simulator ID. llm_interface: Interface to the large language model. item_type: Type of the item to be recommended. Defaults to None. + information_need_tracker: Tracker for the information need. task_definition: Definition of the task to be performed. Defaults to DEFAULT_TASK_DEFINITION. stop_definition: Definition of the stop task. Defaults to DEFAULT_STOP_DEFINITION. persona: Persona of the user. Defaults to None. - information_need_tracker: Tracker to use for the generated - information need. Defaults to a heuristic tracker. """ super().__init__(id, domain, item_collection, information_need_tracker) self.llm_interface = llm_interface diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index 343daf62..f6fdcd9f 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -28,9 +28,9 @@ def __init__( item_collection: ItemCollection, llm_interface: LLMInterface, item_type: str, + information_need_tracker: InformationNeedTracker, task_definition: str = DEFAULT_TASK_DEFINITION, persona: Persona = None, - information_need_tracker: InformationNeedTracker = None, ) -> None: """Initializes the user simulator. @@ -38,11 +38,10 @@ def __init__( id: User simulator ID. llm_interface: Interface to the large language model. item_type: Type of the item to be recommended. Defaults to None. + information_need_tracker: Tracker for the information need. task_definition: Definition of the task to be performed. Defaults to DEFAULT_TASK_DEFINITION. persona: Persona of the user. Defaults to None. - information_need_tracker: Tracker to use for the generated - information need. Defaults to a heuristic tracker. """ super().__init__(id, domain, item_collection, information_need_tracker) self.llm_interface = llm_interface diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index 8c93e84e..fa74ed0f 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -1,17 +1,10 @@ """User simulator abstract class.""" from abc import ABC, abstractmethod -from typing import Optional from dialoguekit.core.annotated_utterance import AnnotatedUtterance from dialoguekit.core.utterance import Utterance from dialoguekit.participant.user import User, UserType -from usersimcrs.information_need_management.information_need import ( - generate_random_information_need, -) -from usersimcrs.information_need_management.information_need_trackers import ( - HeuristicInformationNeedTracker, -) from usersimcrs.information_need_management.information_need_tracker import ( InformationNeedTracker, ) @@ -25,7 +18,7 @@ def __init__( id: str, domain: SimulationDomain, item_collection: ItemCollection, - information_need_tracker: Optional[InformationNeedTracker] = None, + information_need_tracker: InformationNeedTracker, ) -> None: """Initializes the user simulator. @@ -33,26 +26,12 @@ def __init__( id: Simulator ID. domain: Domain. item_collection: Item collection. - information_need_tracker: Tracker to use for the generated - information need. Defaults to None. + information_need_tracker: Tracker for the information need. """ super().__init__(id, UserType.SIMULATOR) self._domain = domain self._item_collection = item_collection self.information_need_tracker = information_need_tracker - self.get_new_information_need() - - def get_new_information_need(self) -> None: - """Generates a new information need.""" - information_need = generate_random_information_need( - self._domain, self._item_collection - ) - if self.information_need_tracker is None: - self.information_need_tracker = HeuristicInformationNeedTracker( - information_need - ) - else: - self.information_need_tracker.set_information_need(information_need) @abstractmethod def _generate_response(self, agent_utterance: Utterance) -> Utterance: diff --git a/usersimcrs/utils/simulation_utils.py b/usersimcrs/utils/simulation_utils.py index 003ffcae..cea93a6b 100644 --- a/usersimcrs/utils/simulation_utils.py +++ b/usersimcrs/utils/simulation_utils.py @@ -23,6 +23,12 @@ from dialoguekit.utils.dialogue_reader import json_to_dialogues from usersimcrs.core.simulation_domain import SimulationDomain +from usersimcrs.information_need_management.information_need import ( + generate_random_information_need, +) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.items.ratings import Ratings from usersimcrs.nlu.llm.llm_dialogue_act_extractor import ( @@ -36,6 +42,30 @@ ) +def _get_information_need_tracker( + config: confuse.Configuration, + domain: SimulationDomain, + item_collection: ItemCollection, +) -> InformationNeedTracker: + """Creates an information need tracker from configuration. + + Args: + config: Configuration of the run. + domain: Domain. + item_collection: Item collection. + + Returns: + Information need tracker. + """ + tracker_config = config["information_need_tracker"].get() + tracker_class = map_path_to_class(tracker_config["class_path"]) + tracker_args = dict(tracker_config.get("args", {})) + tracker_args["information_need"] = generate_random_information_need( + domain, item_collection + ) + return tracker_class(**tracker_args) + + def map_path_to_class(cls_path: str) -> Type: """Maps a class path to a class. @@ -180,6 +210,9 @@ def _get_agenda_based_simulator_config( return { "preference_model": preference_model, "interaction_model": interaction_model, + "information_need_tracker": _get_information_need_tracker( + config, domain, item_collection + ), "nlu": nlu, "nlg": nlg, "domain": domain, @@ -322,6 +355,9 @@ def _get_llm_single_prompt_user_simulator_config( "item_collection": item_collection, "llm_interface": llm_interface, "item_type": item_type, + "information_need_tracker": _get_information_need_tracker( + config, domain, item_collection + ), "task_definition": task_definition, "persona": persona, } From d867c593c6dde722f7f27e205c5c0ea0f9c4adf9 Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Tue, 11 Aug 2026 16:41:33 +0200 Subject: [PATCH 21/22] test fix --- tests/conftest.py | 2 +- usersimcrs/information_need_management/information_need.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fc8147d1..368f0f3b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,7 @@ def information_need() -> InformationNeed: return InformationNeed(target_items, constraints, requests) -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def item_collection(domain: SimulationDomain): """Item collection fixture.""" item_collection = ItemCollection("tests/data/items.db", "test_collection") diff --git a/usersimcrs/information_need_management/information_need.py b/usersimcrs/information_need_management/information_need.py index 914e213b..b37ae3e2 100644 --- a/usersimcrs/information_need_management/information_need.py +++ b/usersimcrs/information_need_management/information_need.py @@ -251,6 +251,6 @@ def to_dict(self) -> Dict[str, Any]: ], "constraints": self.constraints, "requests": list(self.requested_slots.keys()), - "constraint_states": self.constraint_states, - "request_states": self.request_states, + "constraint_states": dict(self.constraint_states), + "request_states": dict(self.request_states), } From 15f9107be43f73c613a1276f26620026b1fb6d7f Mon Sep 17 00:00:00 2001 From: Ksenia Blokhina Date: Mon, 24 Aug 2026 16:29:17 +0200 Subject: [PATCH 22/22] form information need before dialogues --- usersimcrs/run_simulation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/usersimcrs/run_simulation.py b/usersimcrs/run_simulation.py index d3a4572a..164f4eaa 100644 --- a/usersimcrs/run_simulation.py +++ b/usersimcrs/run_simulation.py @@ -31,11 +31,6 @@ def main(config: confuse.Configuration) -> None: config: Configuration generated from YAML configuration file. """ agent_class, agent_config = utils.get_agent_information(config) - ( - simulator_id, - simulator_class, - simulator_config, - ) = utils.get_simulator_information(config) platform = SimulationPlatform(agent_class, agent_config) platform.start() @@ -44,6 +39,11 @@ def main(config: confuse.Configuration) -> None: range(config["num_simulated_dialogues"].get()), desc=" Simulated dialogues", ): + ( + simulator_id, + simulator_class, + simulator_config, + ) = utils.get_simulator_information(config) platform.connect(simulator_id, simulator_class, simulator_config) platform.disconnect(simulator_id)