diff --git a/scripts/datasets/information_need_annotation/information_need_annotator.py b/scripts/datasets/information_need_annotation/information_need_annotator.py index c7076a61..ce5c0368 100644 --- a/scripts/datasets/information_need_annotation/information_need_annotator.py +++ b/scripts/datasets/information_need_annotation/information_need_annotator.py @@ -10,7 +10,9 @@ import os from dialoguekit.core.dialogue import Dialogue -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.llm_interfaces.llm_interface import LLMInterface DEFAULT_INITIAL_PROMPT_MOVIES_FILE = "scripts/datasets/information_need_annotation/information_need_prompt_movies_default.txt" # noqa: E501 diff --git a/tests/conftest.py b/tests/conftest.py index 81a025b9..368f0f3b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,9 @@ from dialoguekit.core.intent import Intent from dialoguekit.utils.dialogue_reader import json_to_dialogues -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item import Item from usersimcrs.items.item_collection import ItemCollection @@ -26,7 +28,7 @@ def domain() -> SimulationDomain: return SimulationDomain(DOMAIN_YAML_FILE) -@pytest.fixture(scope="module") +@pytest.fixture def information_need() -> InformationNeed: """Information need fixture.""" constraints = {"GENRE": "Comedy", "DIRECTOR": "Steven Spielberg"} diff --git a/tests/core/test_information_need.py b/tests/core/test_information_need.py index a339292e..889e1268 100644 --- a/tests/core/test_information_need.py +++ b/tests/core/test_information_need.py @@ -2,7 +2,7 @@ import pytest -from usersimcrs.core.information_need import ( +from usersimcrs.information_need_management.information_need import ( InformationNeed, generate_random_information_need, ) @@ -59,16 +59,12 @@ def test_get_requestable_slots(information_need: InformationNeed) -> None: information_need: Information need. """ assert information_need.get_requestable_slots() == ["PLOT", "RATING"] - information_need.requested_slots["RATING"] = 4.5 + information_need.mark_request_complete("RATING", 4.5) assert information_need.get_requestable_slots() == ["PLOT"] def test_to_dict(information_need: InformationNeed) -> None: - """Test to_dict. - - Args: - information_need: Information need. - """ + """Test to_dict.""" assert information_need.to_dict() == { "target_items": [ { @@ -83,6 +79,14 @@ def test_to_dict(information_need: InformationNeed) -> None: ], "constraints": {"GENRE": "Comedy", "DIRECTOR": "Steven Spielberg"}, "requests": ["PLOT", "RATING"], + "constraint_states": { + "GENRE": InformationNeed.SLOT_STATE_INCOMPLETE, + "DIRECTOR": InformationNeed.SLOT_STATE_INCOMPLETE, + }, + "request_states": { + "PLOT": InformationNeed.SLOT_STATE_INCOMPLETE, + "RATING": InformationNeed.SLOT_STATE_INCOMPLETE, + }, } @@ -140,3 +144,11 @@ def test_from_dict() -> None: loaded_information_need.requested_slots == expected_information_need.requested_slots ) + assert ( + loaded_information_need.constraint_states + == expected_information_need.constraint_states + ) + assert ( + loaded_information_need.request_states + == expected_information_need.request_states + ) diff --git a/tests/simulator/agenda_based/test_interaction_model.py b/tests/simulator/agenda_based/test_interaction_model.py index 57516c28..8aabc08f 100644 --- a/tests/simulator/agenda_based/test_interaction_model.py +++ b/tests/simulator/agenda_based/test_interaction_model.py @@ -6,7 +6,12 @@ from dialoguekit.core.intent import Intent from dialoguekit.core.slot_value_annotation import SlotValueAnnotation from dialoguekit.utils.dialogue_reader import json_to_dialogues -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) +from usersimcrs.information_need_management.information_need_trackers import ( + HeuristicInformationNeedTracker, +) from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.simulator.agenda_based.interaction_model import ( InteractionModel, @@ -29,7 +34,7 @@ def im_crsv1( domain, ANNOTATED_CONVERSATIONS, ) - im.initialize_agenda(information_need) + im.initialize_agenda(HeuristicInformationNeedTracker(information_need)) return im @@ -45,7 +50,9 @@ def test_initialize_with_error(domain: SimulationDomain) -> None: def test_initialize_agenda( im_crsv1: InteractionModel, information_need: InformationNeed ) -> None: - im_crsv1.initialize_agenda(information_need) + im_crsv1.initialize_agenda( + HeuristicInformationNeedTracker(information_need) + ) assert len(im_crsv1.agenda.stack) == 6 assert im_crsv1.agenda.stack[0].intent == im_crsv1.INTENT_START assert im_crsv1.agenda.stack[1].intent == im_crsv1.INTENT_DISCLOSE diff --git a/tests/simulator/llm/test_stop_prompt.py b/tests/simulator/llm/test_stop_prompt.py index acced8e8..132979a1 100644 --- a/tests/simulator/llm/test_stop_prompt.py +++ b/tests/simulator/llm/test_stop_prompt.py @@ -2,7 +2,9 @@ import pytest -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.simulator.llm.prompt.stop_prompt import ( DEFAULT_STOP_DEFINITION, StopPrompt, diff --git a/tests/simulator/llm/test_utterance_generation_prompt.py b/tests/simulator/llm/test_utterance_generation_prompt.py index f1378df8..8a2e7ce0 100644 --- a/tests/simulator/llm/test_utterance_generation_prompt.py +++ b/tests/simulator/llm/test_utterance_generation_prompt.py @@ -4,7 +4,9 @@ from dialoguekit.core import Utterance from dialoguekit.participant import DialogueParticipant -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.simulator.llm.prompt.utterance_generation_prompt import ( DEFAULT_TASK_DEFINITION, UtteranceGenerationPrompt, diff --git a/usersimcrs/core/__init__.py b/usersimcrs/core/__init__.py index f7c76eed..fa93a90a 100644 --- a/usersimcrs/core/__init__.py +++ b/usersimcrs/core/__init__.py @@ -1,9 +1,7 @@ """Module level init for the core classes.""" -from usersimcrs.core.information_need import InformationNeed from usersimcrs.core.simulation_domain import SimulationDomain __all__ = [ - "InformationNeed", "SimulationDomain", ] diff --git a/usersimcrs/core/information_need.py b/usersimcrs/core/information_need.py deleted file mode 100644 index 11636522..00000000 --- a/usersimcrs/core/information_need.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Interface to represent an information need. - -The information need comprises three elements: constraints, requests, and target -items. The constraints specify the slot-value pairs that the item of interest -must satisfy, while the requests specify the slots for which the user wants -information. The target items represent the "ground truth" items that the user -is interested in. -""" - -from __future__ import annotations - -import random -from collections import defaultdict -from typing import Any, Dict, List - -from usersimcrs.core.simulation_domain import SimulationDomain -from usersimcrs.items.item import Item -from usersimcrs.items.item_collection import ItemCollection - - -def generate_random_information_need( - domain: SimulationDomain, item_collection: ItemCollection -) -> InformationNeed: - """Generates a random information need based on the domain. - - It randomly selects one target item and sets constraints and requests slots. - The value of constraints are derived from the target's properties. The - number of constraints and requests are also randomly determined. - - Args: - domain: Domain knowledge. - item_collection: Collection of items. - - Returns: - Information need. - """ - target_item = item_collection.get_random_item() - - constraints = {} - informable_slots = set(domain.get_informable_slots()).intersection( - target_item.properties.keys() - ) - num_constraints = random.randint(1, len(informable_slots)) - for slot in random.sample(list(informable_slots), num_constraints): - constraints[slot] = target_item.get_property(slot) - - requestable_slots = set( - domain.get_requestable_slots() - ).symmetric_difference(constraints.keys()) - num_requests = random.randint(1, len(requestable_slots)) - requests = random.sample(list(requestable_slots), num_requests) - - return InformationNeed([target_item], constraints, requests) - - -class InformationNeed: - def __init__( - self, - target_items: List[Item], - constraints: Dict[str, Any], - requests: List[str], - ) -> None: - """Initializes an information need. - - Args: - target_items: Items that the user is interested in. - constraints: Slot-value pairs representing constraints on the item - of interest. - requests: Slots representing the desired information. - """ - self.target_items = target_items - self.constraints = constraints - self.requested_slots = defaultdict( - None, {slot: None for slot in requests} - ) - - def get_constraint_value(self, slot: str) -> Any: - """Returns the value of a constraint slot. - - Args: - slot: Slot. - - Returns: - Value of the slot. - """ - return self.constraints.get(slot) - - def get_requestable_slots(self) -> List[str]: - """Returns the list of requestable slots.""" - return [ - slot - for slot in self.requested_slots - if not self.requested_slots[slot] - ] - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> InformationNeed: - """Creates information need from a dictionary.""" - target_items = [Item(**item) for item in data["target_items"]] - return cls( - target_items=target_items, - constraints=data["constraints"], - requests=data["requests"], - ) - - def to_dict(self) -> Dict[str, Any]: - """Returns information need as a dictionary.""" - return { - "target_items": [ - {"item_id": item.id, "properties": item.properties} - for item in self.target_items - ], - "constraints": self.constraints, - "requests": list(self.requested_slots.keys()), - } diff --git a/usersimcrs/information_need_management/__init__.py b/usersimcrs/information_need_management/__init__.py new file mode 100644 index 00000000..50d49249 --- /dev/null +++ b/usersimcrs/information_need_management/__init__.py @@ -0,0 +1,19 @@ +"""Information need management.""" + +from usersimcrs.information_need_management.information_need import ( + InformationNeed, + generate_preference_grounded_information_need, + generate_random_information_need, +) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, + SlotUpdate, +) + +__all__ = [ + "InformationNeed", + "InformationNeedTracker", + "SlotUpdate", + "generate_preference_grounded_information_need", + "generate_random_information_need", +] diff --git a/usersimcrs/information_need_management/information_need.py b/usersimcrs/information_need_management/information_need.py new file mode 100644 index 00000000..b37ae3e2 --- /dev/null +++ b/usersimcrs/information_need_management/information_need.py @@ -0,0 +1,256 @@ +"""Interface to represent an information need. + +The information need comprises three elements: constraints, requests, and target +items. The constraints specify the slot-value pairs that the item of interest +must satisfy, while the requests specify the slots for which the user wants +information. The target items represent the "ground truth" items that the user +is interested in. +""" + +from __future__ import annotations + +import random +from collections import defaultdict +from typing import Any, DefaultDict, Dict, List + +from dialoguekit.core.slot_value_annotation import SlotValueAnnotation +from usersimcrs.core.simulation_domain import SimulationDomain +from usersimcrs.items.item import Item +from usersimcrs.items.item_collection import ItemCollection + +from usersimcrs.user_modeling.preference_model import PreferenceModel + + +def generate_random_information_need( + domain: SimulationDomain, item_collection: ItemCollection +) -> InformationNeed: + """Generates a random information need based on the domain. + + It randomly selects one target item and sets constraints and requests slots. + The value of constraints are derived from the target's properties. The + number of constraints and requests are also randomly determined. + + Args: + domain: Domain knowledge. + item_collection: Collection of items. + + Returns: + Information need. + """ + target_item = item_collection.get_random_item() + + constraints = {} + informable_slots = set(domain.get_informable_slots()).intersection( + target_item.properties.keys() + ) + num_constraints = random.randint(1, len(informable_slots)) + for slot in random.sample(list(informable_slots), num_constraints): + constraints[slot] = target_item.get_property(slot) + + requestable_slots = set( + domain.get_requestable_slots() + ).symmetric_difference(constraints.keys()) + num_requests = random.randint(1, len(requestable_slots)) + requests = random.sample(list(requestable_slots), num_requests) + + return InformationNeed([target_item], constraints, requests) + + +def generate_preference_grounded_information_need( + domain: SimulationDomain, + item_collection: ItemCollection, + preference_model: PreferenceModel, +) -> InformationNeed: + """Generates an information need aligned with a preference model. + + The function samples a random number of informable slots, obtains + preference constraints for those slots, then tries to find an item matching + them. If no matching item is found, constraints are relaxed until a match is + found. If no constraints match any items, the information need is returned + without target items. + + Args: + domain: Domain knowledge. + item_collection: Collection of items. + preference_model: Preference model of the simulated user. + + Returns: + Information need. + """ + informable_slots = list(domain.get_informable_slots()) + preferred_slots = random.sample( + informable_slots, random.randint(1, len(informable_slots)) + ) + preferred_constraints = { + slot: value + for slot in preferred_slots + for value, _ in [preference_model.get_slot_preference(slot)] + if value is not None + } + + matching_items = [] + matching_constraints = preferred_constraints + preferred_constraint_items = list(preferred_constraints.items()) + for num_constraints in range(len(preferred_constraint_items), 0, -1): + candidate_constraints = dict( + preferred_constraint_items[:num_constraints] + ) + matching_items = item_collection.get_items_by_properties( + [ + SlotValueAnnotation(slot, value) + for slot, value in candidate_constraints.items() + ] + ) + if matching_items: + matching_constraints = candidate_constraints + break + + target_items = [random.choice(matching_items)] if matching_items else [] + + constraints = ( + { + slot: value + for slot in matching_constraints + for value in [target_items[0].get_property(slot)] + if value is not None + } + if target_items + else matching_constraints + ) + + requestable_slots = [ + slot + for slot in domain.get_requestable_slots() + if slot not in constraints + ] + requests = ( + random.sample( + requestable_slots, random.randint(1, len(requestable_slots)) + ) + if requestable_slots + else [] + ) + + return InformationNeed(target_items, constraints, requests) + + +class InformationNeed: + SLOT_STATE_INCOMPLETE = "incomplete" + SLOT_STATE_ATTEMPTED = "attempted" + SLOT_STATE_COMPLETE = "complete" + + def __init__( + self, + target_items: List[Item], + constraints: Dict[str, Any], + requests: List[str], + constraint_states: Dict[str, str] = {}, + request_states: Dict[str, str] = {}, + ) -> None: + """Initializes an information need. + + Args: + target_items: Items that the user is interested in. + constraints: Slot-value pairs representing constraints on the item + of interest. + requests: Slots representing the desired information. + constraint_states: States for constraints. + request_states: States for requests. + """ + self.target_items = target_items + self.constraints = constraints + self.requested_slots = defaultdict( + None, {slot: None for slot in requests} + ) + self.constraint_states: DefaultDict[str, str] = defaultdict( + lambda: self.SLOT_STATE_INCOMPLETE, + { + slot: constraint_states.get(slot, self.SLOT_STATE_INCOMPLETE) + for slot in constraints + }, + ) + + self.request_states: DefaultDict[str, str] = defaultdict( + lambda: self.SLOT_STATE_INCOMPLETE, + { + slot: request_states.get(slot, self.SLOT_STATE_INCOMPLETE) + for slot in requests + }, + ) + + def get_constraint_value(self, slot: str) -> Any: + """Returns the value of a constraint slot. + + Args: + slot: Slot. + + Returns: + Value of the slot. + """ + return self.constraints.get(slot) + + def get_requestable_slots(self) -> List[str]: + """Returns the list of requestable slots.""" + return [ + slot + for slot in self.requested_slots + if self.request_states.get(slot) != self.SLOT_STATE_COMPLETE + ] + + def _mark_state( + self, states: Dict[str, str], slot: str, state: str + ) -> None: + """Updates the state of a tracked slot. + + Args: + states: Mapping from slots to their current states. + slot: Slot whose state should be updated. + state: New state to assign to the slot. + """ + if slot in states: + states[slot] = state + + def mark_constraint_attempted(self, slot: str) -> None: + """Marks a constraint slot as attempted.""" + self._mark_state( + self.constraint_states, slot, self.SLOT_STATE_ATTEMPTED + ) + + def mark_constraint_complete(self, slot: str) -> None: + """Marks a constraint slot as complete.""" + self._mark_state(self.constraint_states, slot, self.SLOT_STATE_COMPLETE) + + def mark_request_attempted(self, slot: str) -> None: + """Marks a request slot as attempted.""" + self._mark_state(self.request_states, slot, self.SLOT_STATE_ATTEMPTED) + + def mark_request_complete(self, slot: str, value: Any) -> None: + """Marks a request slot as complete with the observed value.""" + if slot in self.request_states: + self.requested_slots[slot] = value + self.request_states[slot] = self.SLOT_STATE_COMPLETE + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> InformationNeed: + """Creates information need from a dictionary.""" + target_items = [Item(**item) for item in data["target_items"]] + return cls( + target_items=target_items, + constraints=data["constraints"], + requests=data["requests"], + constraint_states=data.get("constraint_states", {}), + request_states=data.get("request_states", {}), + ) + + def to_dict(self) -> Dict[str, Any]: + """Returns information need as a dictionary.""" + return { + "target_items": [ + {"item_id": item.id, "properties": item.properties} + for item in self.target_items + ], + "constraints": self.constraints, + "requests": list(self.requested_slots.keys()), + "constraint_states": dict(self.constraint_states), + "request_states": dict(self.request_states), + } diff --git a/usersimcrs/information_need_management/information_need_tracker.py b/usersimcrs/information_need_management/information_need_tracker.py new file mode 100644 index 00000000..3a52ca2c --- /dev/null +++ b/usersimcrs/information_need_management/information_need_tracker.py @@ -0,0 +1,99 @@ +"""Information need tracker. + +It updates the state of the information need during the conversation. That is, +tracking the state (incomplete, attempted, complete) of the constraints and +requests. +""" + +from __future__ import annotations + +import copy +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, List + +from dialoguekit.core.dialogue_act import DialogueAct +from dialoguekit.core.utterance import Utterance + +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) + + +@dataclass +class SlotUpdate: + kind: str + slot: str + status: str + value: Any = None + + +class InformationNeedTracker(ABC): + def __init__(self, information_need: InformationNeed) -> None: + """Initializes the tracker with the information need to update.""" + self.set_information_need(information_need) + + def set_information_need(self, information_need: InformationNeed) -> None: + """Sets the information need tracked by this tracker. + + Args: + information_need: Information need to track. + """ + self._information_need = copy.deepcopy(information_need) + + def get_information_need(self) -> InformationNeed: + """Returns the tracked information need.""" + return self._information_need + + @abstractmethod + def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: + """Builds information-need updates from an utterance. + + Args: + utterance: Utterance to inspect. + + Raises: + NotImplementedError: If not implemented in derived class. + + Returns: + List of slot updates. + """ + raise NotImplementedError + + @abstractmethod + def update_from_agent_dialogue_acts( + self, agent_dialogue_acts: List[DialogueAct] + ) -> List[SlotUpdate]: + """Builds information-need updates from agent dialogue acts. + + Args: + agent_dialogue_acts: Agent dialogue acts to inspect. + + Raises: + NotImplementedError: If not implemented in derived class. + + Returns: + List of slot updates. + """ + raise NotImplementedError + + def apply_updates_to_information_need( + self, updates: List[SlotUpdate] + ) -> None: + """Applies slot updates to the tracked information need. + + Args: + updates: Slot updates to apply to the tracked information need. + """ + for update in updates: + if update.kind == "request": + if update.status == InformationNeed.SLOT_STATE_COMPLETE: + self._information_need.mark_request_complete( + update.slot, update.value + ) + elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: + self._information_need.mark_request_attempted(update.slot) + elif update.status == InformationNeed.SLOT_STATE_COMPLETE: + self._information_need.mark_constraint_complete(update.slot) + elif update.status == InformationNeed.SLOT_STATE_ATTEMPTED: + self._information_need.mark_constraint_attempted(update.slot) diff --git a/usersimcrs/information_need_management/information_need_trackers/__init__.py b/usersimcrs/information_need_management/information_need_trackers/__init__.py new file mode 100644 index 00000000..bd312972 --- /dev/null +++ b/usersimcrs/information_need_management/information_need_trackers/__init__.py @@ -0,0 +1,16 @@ +"""Information need trackers.""" + +from .heuristic_information_need_tracker import ( + HeuristicInformationNeedTracker, +) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, + SlotUpdate, +) + +__all__ = [ + "DEFAULT_UPDATE_PROMPT", + "HeuristicInformationNeedTracker", + "InformationNeedTracker", + "SlotUpdate", +] diff --git a/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py b/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py new file mode 100644 index 00000000..4012af9f --- /dev/null +++ b/usersimcrs/information_need_management/information_need_trackers/heuristic_information_need_tracker.py @@ -0,0 +1,181 @@ +"""Heuristic information need tracker. + +The tracker inspects utterance text and dialogue-act annotations for mentions of +the slots and values contained in an information need. Slot mentions mark the +slot as attempted, while matching target or constraint values mark it complete. +""" + +from __future__ import annotations + +import re +from typing import Any, List + +from dialoguekit.core.dialogue_act import DialogueAct +from dialoguekit.core.utterance import Utterance + +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, + SlotUpdate, +) + + +class HeuristicInformationNeedTracker(InformationNeedTracker): + def _normalize_text(self, text: str) -> str: + """Normalizes text for matching. + + Args: + text: Text to normalize. + + Returns: + Normalized text. + """ + return " ".join(re.sub(r"[_-]", " ", text.lower()).split()) + + def _get_target_slot_values(self, slot: str) -> List[str]: + """Collects normalized target values for a slot. + + Args: + slot: Slot name. + + Returns: + Normalized target values. + """ + normalized_values: List[str] = [] + information_need = self.get_information_need() + for item in information_need.target_items: + value = item.get_property(slot) + if value is None: + continue + values = value if isinstance(value, list) else [value] + normalized_values.extend( + self._normalize_text(str(entry)) for entry in values + ) + return normalized_values + + def _slot_update( + self, kind: str, slot: str, is_complete: bool, value: Any = None + ) -> SlotUpdate: + """Builds a slot update with complete or attempted status. + + Args: + kind: Slot kind, either request or constraint. + slot: Slot name. + is_complete: Whether the slot is complete. + value: Slot value for request completion. + + Returns: + Slot update. + """ + return SlotUpdate( + kind, + slot, + ( + InformationNeed.SLOT_STATE_COMPLETE + if is_complete + else InformationNeed.SLOT_STATE_ATTEMPTED + ), + value, + ) + + def update_from_utterance(self, utterance: Utterance) -> List[SlotUpdate]: + """Builds information-need updates from an utterance. + + Args: + utterance: Utterance to inspect. + + Returns: + List of slot updates. + """ + raw = getattr(utterance, "text", "") + text = self._normalize_text(raw) + + if not text: + return [] + + updates: List[SlotUpdate] = [] + + information_need = self.get_information_need() + for slot in information_need.request_states: + normalized_slot = self._normalize_text(slot) + target_values = self._get_target_slot_values(slot) + value_mentioned = any(value in text for value in target_values) + slot_mentioned = normalized_slot in text + if slot_mentioned or value_mentioned: + updates.append( + self._slot_update( + "request", + slot, + value_mentioned, + raw.strip() if value_mentioned else None, + ) + ) + + if not information_need.target_items and not updates: + attempted_requests = [ + slot + for slot, state in information_need.request_states.items() + if state == InformationNeed.SLOT_STATE_ATTEMPTED + ] + if len(attempted_requests) == 1: + updates.append( + self._slot_update( + "request", attempted_requests[0], True, raw.strip() + ) + ) + + for slot, value in information_need.constraints.items(): + normalized_slot = self._normalize_text(slot) + values = value if isinstance(value, list) else [value] + normalized_values = [self._normalize_text(str(v)) for v in values] + slot_mentioned = normalized_slot in text + value_mentioned = any(v in text for v in normalized_values) + + if slot_mentioned or value_mentioned: + updates.append( + self._slot_update( + "constraint", + slot, + value_mentioned, + ) + ) + + return updates + + def update_from_agent_dialogue_acts( + self, + agent_dialogue_acts: List[DialogueAct], + ) -> List[SlotUpdate]: + """Builds information-need updates from agent dialogue acts. + + Args: + agent_dialogue_acts: Agent dialogue acts to inspect. + + Returns: + List of slot updates. + """ + updates: List[SlotUpdate] = [] + for dialogue_act in agent_dialogue_acts: + for annotation in dialogue_act.annotations: + slot = annotation.slot + value = annotation.value + information_need = self.get_information_need() + if slot in information_need.request_states: + updates.append( + self._slot_update( + "request", slot, value is not None, value + ) + ) + + if slot in information_need.constraint_states: + updates.append( + self._slot_update( + "constraint", + slot, + value + == information_need.get_constraint_value(slot), + ), + ) + return updates diff --git a/usersimcrs/run_simulation.py b/usersimcrs/run_simulation.py index d3a4572a..164f4eaa 100644 --- a/usersimcrs/run_simulation.py +++ b/usersimcrs/run_simulation.py @@ -31,11 +31,6 @@ def main(config: confuse.Configuration) -> None: config: Configuration generated from YAML configuration file. """ agent_class, agent_config = utils.get_agent_information(config) - ( - simulator_id, - simulator_class, - simulator_config, - ) = utils.get_simulator_information(config) platform = SimulationPlatform(agent_class, agent_config) platform.start() @@ -44,6 +39,11 @@ def main(config: confuse.Configuration) -> None: range(config["num_simulated_dialogues"].get()), desc=" Simulated dialogues", ): + ( + simulator_id, + simulator_class, + simulator_config, + ) = utils.get_simulator_information(config) platform.connect(simulator_id, simulator_class, simulator_config) platform.disconnect(simulator_id) diff --git a/usersimcrs/simulator/agenda_based/agenda.py b/usersimcrs/simulator/agenda_based/agenda.py index 6a366ca8..5700daf4 100644 --- a/usersimcrs/simulator/agenda_based/agenda.py +++ b/usersimcrs/simulator/agenda_based/agenda.py @@ -12,7 +12,9 @@ from dialoguekit.core.dialogue_act import DialogueAct from dialoguekit.core.intent import Intent from dialoguekit.core.slot_value_annotation import SlotValueAnnotation -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) class Agenda: diff --git a/usersimcrs/simulator/agenda_based/agenda_based_simulator.py b/usersimcrs/simulator/agenda_based/agenda_based_simulator.py index e4659b58..fdd3b604 100644 --- a/usersimcrs/simulator/agenda_based/agenda_based_simulator.py +++ b/usersimcrs/simulator/agenda_based/agenda_based_simulator.py @@ -10,6 +10,9 @@ from usersimcrs.dialogue_management.dialogue_state_tracker import ( DialogueStateTracker, ) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.items.ratings import Ratings from usersimcrs.simulator.agenda_based.interaction_model import InteractionModel @@ -25,6 +28,7 @@ def __init__( item_collection: ItemCollection, preference_model: PreferenceModel, interaction_model: InteractionModel, + information_need_tracker: InformationNeedTracker, nlu: NLU, nlg: ConditionalNLG, ratings: Ratings, @@ -37,14 +41,20 @@ def __init__( item_collection: Item collection. preference_model: Preference model. interaction_model: Interaction model. + information_need_tracker: Tracker for the information need. nlu: NLU module performing dialogue act extraction. nlg: NLG module generating textual responses. ratings: Historical ratings. """ - super().__init__(id=id, domain=domain, item_collection=item_collection) + super().__init__( + id=id, + domain=domain, + item_collection=item_collection, + information_need_tracker=information_need_tracker, + ) self._preference_model = preference_model self._interaction_model = interaction_model - self._interaction_model.initialize_agenda(self.information_need) + self._interaction_model.initialize_agenda(self.information_need_tracker) self._nlu = nlu self._nlg = nlg self._dialogue_state_tracker = DialogueStateTracker() @@ -92,7 +102,7 @@ def generate_response( # Update agenda based on the agent's dialogue acts. self._interaction_model.update_agenda( - self.information_need, + self.information_need_tracker, self._preference_model, self._item_collection, ) diff --git a/usersimcrs/simulator/agenda_based/interaction_model.py b/usersimcrs/simulator/agenda_based/interaction_model.py index 97318b90..99c3c6c3 100644 --- a/usersimcrs/simulator/agenda_based/interaction_model.py +++ b/usersimcrs/simulator/agenda_based/interaction_model.py @@ -20,7 +20,12 @@ from dialoguekit.core.intent import Intent from dialoguekit.core.slot_value_annotation import SlotValueAnnotation from dialoguekit.participant import DialogueParticipant -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.dialogue_management.dialogue_state_tracker import ( DialogueStateTracker, @@ -209,14 +214,16 @@ def _get_agent_user_interactions( user_dialogue_acts = [] return agent_user_interactions - def initialize_agenda(self, information_need: InformationNeed): + def initialize_agenda( + self, information_need_tracker: InformationNeedTracker + ): """Initializes user agenda. Args: - information_need: Information need. + information_need_tracker: Information need tracker. """ self.agenda = Agenda( - information_need, + information_need_tracker.get_information_need(), self.INTENT_DISCLOSE, # type: ignore[attr-defined] self.INTENT_INQUIRE, # type: ignore[attr-defined] self.INTENT_STOP, # type: ignore[attr-defined] @@ -300,7 +307,7 @@ def get_next_dialogue_acts(self, n: int = 1) -> List[DialogueAct]: def update_agenda( self, - information_need: InformationNeed, + information_need_tracker: InformationNeedTracker, preference_model: PreferenceModel, item_collection: ItemCollection, ) -> None: @@ -311,10 +318,16 @@ def update_agenda( neither. Once the push operations are done, we clean the stack. Args: - information_need: Information need. + information_need_tracker: Information need tracker. """ + information_need = information_need_tracker.get_information_need() current_state = self.dialogue_state_tracker.get_current_state() agent_dialogue_acts = current_state.agent_dialogue_acts[-1] + information_need_tracker.apply_updates_to_information_need( + information_need_tracker.update_from_agent_dialogue_acts( + agent_dialogue_acts + ), + ) user_dialogue_acts = [] for dialogue_act in agent_dialogue_acts: if self.is_agent_intent_elicit(dialogue_act.intent): @@ -424,6 +437,16 @@ def _generate_elicit_response_dialogue_acts( [SlotValueAnnotation(elicited_slot, elicited_value)], ) ) + if elicited_slot in information_need.constraint_states: + constraint_value = information_need.get_constraint_value( + elicited_slot + ) + if elicited_value == constraint_value: + information_need.mark_constraint_complete(elicited_slot) + else: + information_need.mark_constraint_attempted( + elicited_slot + ) else: # Agent is asking about value preferences on a given slot, e.g., # "What movie genre would you prefer?" The value is taken either @@ -451,6 +474,8 @@ def _generate_elicit_response_dialogue_acts( annotations, ) ) + if elicited_slot in information_need.constraint_states: + information_need.mark_constraint_complete(elicited_slot) else: user_dialogue_acts.append(DialogueAct(self.INTENT_DONT_KNOW)) # type: ignore[attr-defined] # noqa @@ -528,6 +553,7 @@ def _generate_inquire_response_dialogue_acts( slot = slot_value_annotation.slot if slot_value_annotation.value is None: if slot in information_need.get_requestable_slots(): + information_need.mark_request_attempted(slot) user_dialogue_acts.append( DialogueAct( self.INTENT_YES, [SlotValueAnnotation(slot)] # type: ignore[attr-defined] # noqa @@ -548,6 +574,8 @@ def _generate_inquire_response_dialogue_acts( slot = random.choice(requestable_slots) else: slot = random.choice(self._domain.get_requestable_slots()) + if slot in information_need.request_states: + information_need.mark_request_attempted(slot) user_dialogue_acts.append( DialogueAct(self.INQUIRE, [SlotValueAnnotation(slot)]) # type: ignore[attr-defined] # noqa ) @@ -594,6 +622,7 @@ def _sample_next_user_dialogue_acts( slot, value = random.choice( list(information_need.constraints.items()) ) + information_need.mark_constraint_complete(slot) user_dialogue_acts.append( DialogueAct( @@ -601,9 +630,14 @@ def _sample_next_user_dialogue_acts( ) ) elif sampled_intent == self.INTENT_INQUIRE: # type: ignore[attr-defined] # noqa - slot = random.choice(information_need.get_requestable_slots()) - if not slot: - slot = random.choice(self._domain.get_requestable_slots()) + requestable_slots = information_need.get_requestable_slots() + slot = ( + random.choice(requestable_slots) + if requestable_slots + else random.choice(self._domain.get_requestable_slots()) + ) + if slot in information_need.request_states: + information_need.mark_request_attempted(slot) user_dialogue_acts.append( DialogueAct(sampled_intent, [SlotValueAnnotation(slot)]) ) diff --git a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py index 4036a57d..de6d9be6 100644 --- a/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_dual_prompt_user_simulator.py @@ -8,6 +8,9 @@ from dialoguekit.core.utterance import Utterance from dialoguekit.participant import DialogueParticipant from usersimcrs.core.simulation_domain import SimulationDomain +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface from usersimcrs.simulator.llm.prompt.stop_prompt import ( @@ -30,6 +33,7 @@ def __init__( item_collection: ItemCollection, llm_interface: LLMInterface, item_type: str, + information_need_tracker: InformationNeedTracker, task_definition: str = DEFAULT_TASK_DEFINITION, stop_definition: str = DEFAULT_STOP_DEFINITION, persona: Persona = None, @@ -40,19 +44,26 @@ def __init__( id: User simulator ID. llm_interface: Interface to the large language model. item_type: Type of the item to be recommended. Defaults to None. + information_need_tracker: Tracker for the information need. task_definition: Definition of the task to be performed. Defaults to DEFAULT_TASK_DEFINITION. stop_definition: Definition of the stop task. Defaults to DEFAULT_STOP_DEFINITION. persona: Persona of the user. Defaults to None. """ - super().__init__(id, domain, item_collection) + super().__init__(id, domain, item_collection, information_need_tracker) self.llm_interface = llm_interface self.generation_prompt = UtteranceGenerationPrompt( - self.information_need, item_type, task_definition, persona + self.information_need_tracker.get_information_need(), + item_type, + task_definition, + persona, ) self.stop_prompt = StopPrompt( - self.information_need, item_type, stop_definition, persona + self.information_need_tracker.get_information_need(), + item_type, + stop_definition, + persona, ) def _generate_response(self, agent_utterance: Utterance) -> Utterance: @@ -64,6 +75,9 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance(agent_utterance) + ) self.generation_prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) @@ -84,4 +98,10 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: self.generation_prompt.update_prompt_context( user_utterance, DialogueParticipant.USER ) + self.stop_prompt.update_prompt_context( + user_utterance, DialogueParticipant.USER + ) + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance(user_utterance) + ) return user_utterance diff --git a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py index 3d413242..f6fdcd9f 100644 --- a/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py +++ b/usersimcrs/simulator/llm/llm_single_prompt_user_simulator.py @@ -7,6 +7,9 @@ from dialoguekit.core.utterance import Utterance from dialoguekit.participant import DialogueParticipant from usersimcrs.core.simulation_domain import SimulationDomain +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.llm_interfaces.llm_interface import LLMInterface from usersimcrs.simulator.llm.prompt.utterance_generation_prompt import ( @@ -25,6 +28,7 @@ def __init__( item_collection: ItemCollection, llm_interface: LLMInterface, item_type: str, + information_need_tracker: InformationNeedTracker, task_definition: str = DEFAULT_TASK_DEFINITION, persona: Persona = None, ) -> None: @@ -34,14 +38,18 @@ def __init__( id: User simulator ID. llm_interface: Interface to the large language model. item_type: Type of the item to be recommended. Defaults to None. + information_need_tracker: Tracker for the information need. task_definition: Definition of the task to be performed. Defaults to DEFAULT_TASK_DEFINITION. persona: Persona of the user. Defaults to None. """ - super().__init__(id, domain, item_collection) + super().__init__(id, domain, item_collection, information_need_tracker) self.llm_interface = llm_interface self.prompt = UtteranceGenerationPrompt( - self.information_need, item_type, task_definition, persona + self.information_need_tracker.get_information_need(), + item_type, + task_definition, + persona, ) def _generate_response(self, agent_utterance: Utterance) -> Utterance: @@ -53,6 +61,9 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: Returns: User utterance. """ + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance(agent_utterance) + ) self.prompt.update_prompt_context( agent_utterance, DialogueParticipant.AGENT ) @@ -60,4 +71,7 @@ def _generate_response(self, agent_utterance: Utterance) -> Utterance: self.prompt.update_prompt_context( user_utterance, DialogueParticipant.USER ) + self.information_need_tracker.apply_updates_to_information_need( + self.information_need_tracker.update_from_utterance(user_utterance) + ) return user_utterance diff --git a/usersimcrs/simulator/llm/prompt/prompt.py b/usersimcrs/simulator/llm/prompt/prompt.py index 619130ab..df24c990 100644 --- a/usersimcrs/simulator/llm/prompt/prompt.py +++ b/usersimcrs/simulator/llm/prompt/prompt.py @@ -4,7 +4,9 @@ from dialoguekit.core.utterance import Utterance from dialoguekit.participant.participant import DialogueParticipant -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.user_modeling.persona import Persona diff --git a/usersimcrs/simulator/llm/prompt/stop_prompt.py b/usersimcrs/simulator/llm/prompt/stop_prompt.py index c776e27b..f29ebb48 100644 --- a/usersimcrs/simulator/llm/prompt/stop_prompt.py +++ b/usersimcrs/simulator/llm/prompt/stop_prompt.py @@ -1,6 +1,8 @@ """Define the prompt for stopping the conversation.""" -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.simulator.llm.prompt.prompt import Prompt from usersimcrs.user_modeling.persona import Persona diff --git a/usersimcrs/simulator/llm/prompt/utterance_generation_prompt.py b/usersimcrs/simulator/llm/prompt/utterance_generation_prompt.py index 6f65a4e6..c18e2e2c 100644 --- a/usersimcrs/simulator/llm/prompt/utterance_generation_prompt.py +++ b/usersimcrs/simulator/llm/prompt/utterance_generation_prompt.py @@ -11,7 +11,9 @@ from dialoguekit.core.utterance import Utterance from dialoguekit.participant.participant import DialogueParticipant -from usersimcrs.core.information_need import InformationNeed +from usersimcrs.information_need_management.information_need import ( + InformationNeed, +) from usersimcrs.simulator.llm.prompt.prompt import Prompt from usersimcrs.user_modeling.persona import Persona diff --git a/usersimcrs/simulator/user_simulator.py b/usersimcrs/simulator/user_simulator.py index af238c4e..fa74ed0f 100644 --- a/usersimcrs/simulator/user_simulator.py +++ b/usersimcrs/simulator/user_simulator.py @@ -5,7 +5,9 @@ from dialoguekit.core.annotated_utterance import AnnotatedUtterance from dialoguekit.core.utterance import Utterance from dialoguekit.participant.user import User, UserType -from usersimcrs.core.information_need import generate_random_information_need +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.core.simulation_domain import SimulationDomain from usersimcrs.items.item_collection import ItemCollection @@ -16,18 +18,20 @@ def __init__( id: str, domain: SimulationDomain, item_collection: ItemCollection, + information_need_tracker: InformationNeedTracker, ) -> None: - """Initializes the user simulator.""" + """Initializes the user simulator. + + Args: + id: Simulator ID. + domain: Domain. + item_collection: Item collection. + information_need_tracker: Tracker for the information need. + """ super().__init__(id, UserType.SIMULATOR) self._domain = domain self._item_collection = item_collection - self.get_new_information_need() - - def get_new_information_need(self) -> None: - """Generates a new information need.""" - self.information_need = generate_random_information_need( - self._domain, self._item_collection - ) + self.information_need_tracker = information_need_tracker @abstractmethod def _generate_response(self, agent_utterance: Utterance) -> Utterance: diff --git a/usersimcrs/utils/simulation_utils.py b/usersimcrs/utils/simulation_utils.py index 003ffcae..cea93a6b 100644 --- a/usersimcrs/utils/simulation_utils.py +++ b/usersimcrs/utils/simulation_utils.py @@ -23,6 +23,12 @@ from dialoguekit.utils.dialogue_reader import json_to_dialogues from usersimcrs.core.simulation_domain import SimulationDomain +from usersimcrs.information_need_management.information_need import ( + generate_random_information_need, +) +from usersimcrs.information_need_management.information_need_tracker import ( + InformationNeedTracker, +) from usersimcrs.items.item_collection import ItemCollection from usersimcrs.items.ratings import Ratings from usersimcrs.nlu.llm.llm_dialogue_act_extractor import ( @@ -36,6 +42,30 @@ ) +def _get_information_need_tracker( + config: confuse.Configuration, + domain: SimulationDomain, + item_collection: ItemCollection, +) -> InformationNeedTracker: + """Creates an information need tracker from configuration. + + Args: + config: Configuration of the run. + domain: Domain. + item_collection: Item collection. + + Returns: + Information need tracker. + """ + tracker_config = config["information_need_tracker"].get() + tracker_class = map_path_to_class(tracker_config["class_path"]) + tracker_args = dict(tracker_config.get("args", {})) + tracker_args["information_need"] = generate_random_information_need( + domain, item_collection + ) + return tracker_class(**tracker_args) + + def map_path_to_class(cls_path: str) -> Type: """Maps a class path to a class. @@ -180,6 +210,9 @@ def _get_agenda_based_simulator_config( return { "preference_model": preference_model, "interaction_model": interaction_model, + "information_need_tracker": _get_information_need_tracker( + config, domain, item_collection + ), "nlu": nlu, "nlg": nlg, "domain": domain, @@ -322,6 +355,9 @@ def _get_llm_single_prompt_user_simulator_config( "item_collection": item_collection, "llm_interface": llm_interface, "item_type": item_type, + "information_need_tracker": _get_information_need_tracker( + config, domain, item_collection + ), "task_definition": task_definition, "persona": persona, }