diff --git a/docs/configure-rails/colang/colang-2/language-reference/make-use-of-llms.mdx b/docs/configure-rails/colang/colang-2/language-reference/make-use-of-llms.mdx index 94c21b46f5..f317b1c031 100644 --- a/docs/configure-rails/colang/colang-2/language-reference/make-use-of-llms.mdx +++ b/docs/configure-rails/colang/colang-2/language-reference/make-use-of-llms.mdx @@ -49,6 +49,20 @@ model: meta/llama-3.1-70b-instruct ``` To support other models, you would need to create a set of new [template prompts](https://github.com/NVIDIA-NeMo/Guardrails/tree/develop/nemoguardrails/llm/prompts) that consider the specific capabilities and the API of the model and add them to your bot configuration. +For `engine: openai` with GPT-5 or GPT-6 model names, dedicated chat templates cover +user-intent detection, combined intent and bot-action generation, interaction +continuation, and value generation. These tasks use the `main` model unless you +configure a task-specific model. The default OpenAI-compatible framework requires +`OPENAI_API_KEY` and no optional integration extra. These templates do not change +the prompts for generating whole flows from names or natural-language descriptions. + +Generated bot actions must use the `bot action:` format shown in the templates. +The parser accepts straight or paired curly double quotes as string delimiters, +preserves quotes inside the string, and stops before subsequent dialogue turns. +An answer with no bot action raises an `LlmResponseError` instead of creating an +empty continuation flow. Custom prompts should request only the next turn; +provider-side stop sequences are not available on every model. + ## Natural Language Description (NLD) One of the main LLM generation mechanisms in Colang is the so-called Natural Language Description (NLD) in combination with the "generation" operator `...`. diff --git a/docs/configure-rails/yaml-schema/prompt-configuration.mdx b/docs/configure-rails/yaml-schema/prompt-configuration.mdx index f5cf9e6fd2..1e9ca70f38 100644 --- a/docs/configure-rails/yaml-schema/prompt-configuration.mdx +++ b/docs/configure-rails/yaml-schema/prompt-configuration.mdx @@ -317,11 +317,19 @@ The library includes predefined prompts for these models: - `openai/gpt-3.5-turbo-instruct` - `openai/gpt-3.5-turbo` - `openai/gpt-4` +- `openai/gpt-5` and `openai/gpt-6` model families - `databricks/dolly-v2-3b` - `cohere/command` - `cohere/command-light` - `cohere/command-light-nightly` +For Colang 1, the GPT-5 and GPT-6 prompt families use chat messages for intent +classification, next-step generation, bot messages, single-call dialog generation, +and value generation. Intent prompts request structured output instead of a spoken +reply. Single-call generation requests two intent lines followed by the bot message; +the parser also accepts an optional leading quoted user-message echo in buffered +and streaming modes. Custom prompts can override these templates using `models`. + Predefined prompts are continuously evaluated and improved. Test and customize prompts for your specific use case before deploying to production. diff --git a/nemoguardrails/actions/llm/generation.py b/nemoguardrails/actions/llm/generation.py index 723631b28d..7d67a26ff0 100644 --- a/nemoguardrails/actions/llm/generation.py +++ b/nemoguardrails/actions/llm/generation.py @@ -121,6 +121,19 @@ def _streaming_pattern_for(output_parser, *, include_bot_message_parser: bool): return (' "', " “"), ('"', "”") +def _remove_echoed_user_message(completion: str) -> Tuple[str, bool]: + """Remove an optional quoted user-message line before the generated intents.""" + lines = completion.splitlines(keepends=True) + for index, line in enumerate(lines): + line = line.strip() + if not line or line.startswith("#"): + continue + if re.fullmatch(r'(?:user|User message:)\s+["“].*["”]', line, flags=re.IGNORECASE): + return "".join(lines[index + 1 :]), True + break + return completion, False + + @dataclass class SingleCallPayload: """The bot intent/message events computed by the single-call generation. @@ -1433,6 +1446,11 @@ async def generate_intent_steps_message( ) _streaming_handler.set_producer_task(producer_task) result = await _streaming_handler.wait_top_k_nonempty_lines(k=2) + result, echoed_user_message = _remove_echoed_user_message(result) + if echoed_user_message: + # The echo consumed one of the two buffered intent lines. + bot_intent_line = await _streaming_handler.wait_top_k_nonempty_lines(k=1) + result += "\n" + bot_intent_line # Moving forward we need to set the expected pattern to correctly # parse the message. @@ -1452,6 +1470,7 @@ async def generate_intent_steps_message( "temperature": self.config.lowest_temperature, } result = (await llm_call(generation_llm, prompt, llm_params=additional_params)).content + result, _ = _remove_echoed_user_message(result) # Parse the output using the associated parser result = self.llm_task_manager.parse_task_output(Task.GENERATE_INTENT_STEPS_MESSAGE, output=result) diff --git a/nemoguardrails/actions/llm/utils.py b/nemoguardrails/actions/llm/utils.py index 37a72d2271..3f4fc5d134 100644 --- a/nemoguardrails/actions/llm/utils.py +++ b/nemoguardrails/actions/llm/utils.py @@ -438,10 +438,52 @@ def get_first_bot_intent(strings: List[str]) -> Optional[str]: return None +def _normalize_action_quotes(action: str) -> str: + """Normalize generated curly string delimiters, preserving quoted content.""" + if "“" not in action: + return action + result = [] + delimiter = None + curly_depth = 0 + escaped = False + for char in action: + if escaped: + result.append(char) + escaped = False + continue + if char == "\\" and delimiter is not None: + result.append(char) + escaped = True + continue + if delimiter == "“": + if char == "“": + curly_depth += 1 + elif char == "”": + curly_depth -= 1 + if curly_depth == 0: + delimiter = None + result.append('"') + continue + elif char == '"': + result.append('\\"') + continue + elif delimiter is not None: + if char == delimiter: + delimiter = None + elif char in ('"', "'", "“"): + delimiter = char + if char == "“": + curly_depth = 1 + result.append('"') + continue + result.append(char) + return "".join(result) + + def _has_unclosed_quote(s: str) -> bool: """Check if a string has an unclosed double quote (ignoring escaped quotes).""" # Strip escaped backslashes first so that \\" is not misread as \" (escaped quote). - return s.replace("\\\\", "").replace('\\"', "").count('"') % 2 == 1 + return _normalize_action_quotes(s).replace("\\\\", "").replace('\\"', "").count('"') % 2 == 1 _MAX_QUOTE_CONTINUATION_LINES = 50 @@ -458,7 +500,7 @@ def get_first_bot_action(strings: List[str]) -> Optional[str]: if action and _has_unclosed_quote(action): if continuation_lines >= _MAX_QUOTE_CONTINUATION_LINES: # Safety bound: stop collecting to avoid absorbing all input. - return action + return _normalize_action_quotes(action) action += "\\n" + string continuation_lines += 1 if not _has_unclosed_quote(action): @@ -467,7 +509,7 @@ def get_first_bot_action(strings: List[str]) -> Optional[str]: if string.startswith("bot action: "): if action != "": action += "\n" - action += string.replace("bot action: ", "") + action += string.removeprefix("bot action: ") action_started = True elif (string.startswith(" and") or string.startswith(" or")) and action_started: action = action + string @@ -475,8 +517,8 @@ def get_first_bot_action(strings: List[str]) -> Optional[str]: action_started = False continue elif action != "": - return action - return action + return _normalize_action_quotes(action) + return _normalize_action_quotes(action) def escape_flow_name(name: str) -> str: diff --git a/nemoguardrails/actions/v2_x/generation.py b/nemoguardrails/actions/v2_x/generation.py index 9477b11ad7..47544fff11 100644 --- a/nemoguardrails/actions/v2_x/generation.py +++ b/nemoguardrails/actions/v2_x/generation.py @@ -389,7 +389,7 @@ async def generate_user_intent_and_bot_action( bot_intent = get_first_bot_intent(result.splitlines()) bot_action = get_first_bot_action(result.splitlines()) - if bot_action is None: + if not bot_action or not bot_action.strip(): raise LlmResponseError(f"Issue with LLM response: {result}") user_intent = escape_flow_name(user_intent.strip(" ")) @@ -705,7 +705,7 @@ async def generate_flow_continuation( bot_intent = get_first_bot_intent(result.splitlines()) bot_action = get_first_bot_action(result.splitlines()) - if bot_action is None: + if not bot_action or not bot_action.strip(): raise LlmResponseError(f"Issue with LLM response: {result}") if bot_intent: @@ -719,7 +719,7 @@ async def generate_flow_continuation( return { "name": flow_name, "parameters": flow_parameters, - "body": f'@meta(bot_intent="{bot_intent}")\n' + f"flow {flow_name}\n" + f" {bot_action}", + "body": f'@meta(bot_intent="{bot_intent}")\n' + f"flow {flow_name}\n" + textwrap.indent(bot_action, " "), } @action(name="CreateFlowAction", is_system_action=True, execute_async=True) @@ -738,7 +738,7 @@ async def create_flow( flow_name = f"_dynamic_{uuid} {name}" # TODO: parse potential parameters from flow name with a regex - body = f"flow {flow_name}\n " + body + body = f"flow {flow_name}\n" + textwrap.indent(body, " ") if decorators: body = decorators + "\n" + body diff --git a/nemoguardrails/llm/prompts/openai-colang1.yml b/nemoguardrails/llm/prompts/openai-colang1.yml new file mode 100644 index 0000000000..66e9aa9e4a --- /dev/null +++ b/nemoguardrails/llm/prompts/openai-colang1.yml @@ -0,0 +1,139 @@ +prompts: + - task: generate_user_intent + models: &models + - openai/gpt-5 + - openai/gpt-6 + messages: + - type: system + content: |- + {{ general_instructions }} + + Classify the latest user message into a Colang 1 user intent. + Prefer a matching intent from the examples. If none matches, generate + a short intent name that describes the user's request. + Return exactly one line: User intent: + Do not answer the user, repeat their message, or include explanations + or Markdown fences. + - type: user + content: |- + Example conversation: + {{ sample_conversation | verbose_v1 }} + + Intent examples: + {{ examples | verbose_v1 }} + + Candidate intents: {{ potential_user_intents }} + + Conversation: + {{ history | colang | verbose_v1 }} + output_parser: user_intent + + - task: generate_next_steps + models: *models + messages: + - type: system + content: |- + {{ general_instructions }} + + Generate the next Colang 1 flow steps for the latest user intent, + following the matching examples. For a bot response, return only: + Bot intent: + When an example calls for actions or other flow steps, return those + Colang steps instead. Do not write the spoken answer, repeat the user + intent, or include explanations or Markdown fences. + - type: user + content: |- + Example conversation: + {{ sample_conversation | remove_text_messages | verbose_v1 }} + + Flow examples: + {{ examples | remove_text_messages | verbose_v1 }} + + Conversation: + {{ history | colang | remove_text_messages | verbose_v1 }} + output_parser: verbose_v1 + + - task: generate_bot_message + models: *models + messages: + - type: system + content: |- + {{ general_instructions }} + + Write the spoken response for the latest bot intent in the conversation. + Return only: Bot message: "" + Use straight double quotes around the response. Do not repeat intents, + invent another turn, or include explanations or Markdown fences. + - type: user + content: |- + Example conversation: + {{ sample_conversation | verbose_v1 }} + + Bot response examples: + {{ examples | verbose_v1 }} + + {% if relevant_chunks %} + Reference information: + {{ relevant_chunks }} + {% endif %} + + Conversation: + {{ history | colang | verbose_v1 }} + output_parser: bot_message + + - task: generate_intent_steps_message + models: *models + messages: + - type: system + content: |- + {{ general_instructions }} + + Generate the next turn of a Colang 1 interaction, using exactly this format: + User intent: + Bot intent: + Bot message: "" + + Prefer a matching user intent from the examples. If none matches, + generate a short intent name that describes the user's request. + Use straight double quotes around the response. Generate only these + two intents and the bot message. Do not repeat the user message, + invent another turn, or include explanations or Markdown fences. + - type: user + content: |- + Example conversation: + {{ sample_conversation | verbose_v1 }} + + Flow examples: + {{ examples | verbose_v1 }} + + Candidate intents: {{ potential_user_intents }} + + {% if relevant_chunks %} + Reference information: + {{ relevant_chunks }} + {% endif %} + + Conversation: + {{ history | colang | verbose_v1 }} + output_parser: verbose_v1 + + - task: generate_value + models: *models + messages: + - type: system + content: |- + {{ general_instructions }} + + Generate a value for the variable {{ var_name }} following the instruction. + Return only a Python literal on one line, with no assignment, explanation, + or Markdown fences. Use True and False for booleans and straight quotes + for strings. + - type: user + content: |- + Flow examples: + {{ examples }} + + Conversation: + {{ history | colang }} + + Instruction: {{ instructions }} diff --git a/nemoguardrails/llm/prompts/openai-colang2.yml b/nemoguardrails/llm/prompts/openai-colang2.yml new file mode 100644 index 0000000000..79975c886b --- /dev/null +++ b/nemoguardrails/llm/prompts/openai-colang2.yml @@ -0,0 +1,106 @@ +prompts: + - task: generate_user_intent_from_user_action + models: &models + - openai/gpt-5 + - openai/gpt-6 + messages: + - type: system + content: |- + {{ general_instructions }} + + Classify the user action into a Colang 2 user intent. + Prefer a matching intent from the examples. Return exactly one line: + user intent: + Do not include explanations, Markdown fences, or a bot response. + - type: user + content: |- + Intent examples: + {{ examples }} + + Conversation: + {{ history | colang }} + + user action: {{ user_action }} + + - task: generate_user_intent_and_bot_action_from_user_action + models: *models + messages: + - type: system + content: |- + {{ general_instructions }} + + Generate the next turn of a Colang 2 interaction, using exactly this format: + user intent: + bot intent: + bot action: + + Prefer a matching user intent from the examples. For a spoken response, + use bot action: bot say "" with straight double quote delimiters. + Escape double quotes within the response with a backslash. + Put the answer inside the bot say string, never outside the action. + Generate only this turn. Do not invent a subsequent user turn or include + explanations, Markdown fences, or additional dialogue. + - type: user + content: |- + Example conversation: + {{ sample_conversation }} + + Intent examples: + {{ examples }} + + Conversation: + {{ history | colang }} + + {% if context.relevant_chunks %} + Reference information: + {{ context.relevant_chunks }} + {% endif %} + + user action: {{ user_action }} + + - task: generate_flow_continuation + models: *models + messages: + - type: system + content: |- + {{ general_instructions }} + + Generate the next bot turn of a Colang 2 interaction, using exactly this format: + bot intent: + bot action: + + For a spoken response, use bot action: bot say "" with straight + double quote delimiters. Escape double quotes within the response with + a backslash. Put the answer inside the bot say string, never outside the action. + Generate only this bot turn. Do not invent a subsequent user turn or include + explanations, Markdown fences, or additional dialogue. + - type: user + content: |- + Example conversation: + {{ sample_conversation }} + + Conversation: + {{ history | colang }} + + {% if context.relevant_chunks %} + Reference information: + {{ context.relevant_chunks }} + {% endif %} + + - task: generate_value_from_instruction + models: *models + messages: + - type: system + content: |- + {{ general_instructions }} + + Generate a value for the variable {{ var_name }} following the instruction. + Return only a Python literal on one line, with no assignment, explanation, + or Markdown fences. Use True and False for booleans and straight quotes + for strings. + - type: user + content: |- + Conversation: + {{ history | colang }} + + Instruction: {{ instructions }} diff --git a/nemoguardrails/streaming.py b/nemoguardrails/streaming.py index 47aa339b61..6125f507e5 100644 --- a/nemoguardrails/streaming.py +++ b/nemoguardrails/streaming.py @@ -158,6 +158,7 @@ async def wait_top_k_nonempty_lines(self, k: int): and remove them from the buffer. """ self.k = k + self.top_k_nonempty_lines_event.clear() self._notify_buffer_ready() await self._wait_for_event_or_producer(self.top_k_nonempty_lines_event) @@ -177,7 +178,7 @@ async def wait_top_k_nonempty_lines(self, k: int): def _notify_buffer_ready(self): lines = [line.strip() for line in self.buffer.split("\n")] lines = [line for line in lines if len(line) > 0 and line[0] != "#"] - if len(lines) > self.k > 0: + if self._buffered_end_of_stream or len(lines) > self.k > 0: self.top_k_nonempty_lines_event.set() async def enable_buffering(self): @@ -262,6 +263,7 @@ async def _process( self._notify_buffer_ready() elif chunk is END_OF_STREAM: self._buffered_end_of_stream = True + self._notify_buffer_ready() else: prev_completion = self.completion diff --git a/tests/colang/parser/v2_x/test_multiline_bot_action.py b/tests/colang/parser/v2_x/test_multiline_bot_action.py index 7cab06a4d6..16cca86a53 100644 --- a/tests/colang/parser/v2_x/test_multiline_bot_action.py +++ b/tests/colang/parser/v2_x/test_multiline_bot_action.py @@ -15,6 +15,8 @@ """Tests for multi-line bot say quoted string handling in get_first_bot_action.""" +import pytest + from nemoguardrails.actions.llm.utils import ( _MAX_QUOTE_CONTINUATION_LINES, _has_unclosed_quote, @@ -22,6 +24,37 @@ ) +@pytest.mark.parametrize( + ("completion", "expected"), + [ + ("bot say “Hello!”", 'bot say "Hello!"'), + ("bot say “Hello\nworld!”", 'bot say "Hello\\nworld!"'), + ("bot say “She said “hello”.”", 'bot say "She said “hello”."'), + ('bot say “She said "hello".”', 'bot say "She said \\"hello\\"."'), + ('bot say "She said “hello”."', 'bot say "She said “hello”."'), + ("bot say 'She said “hello”.'", "bot say 'She said “hello”.'"), + ('bot say “It’s fine.” and bot gesture "wave"', 'bot say "It’s fine." and bot gesture "wave"'), + ("bot say “literal bot action: text”", 'bot say "literal bot action: text"'), + ], +) +def test_generated_smart_quote_delimiters(completion, expected): + from nemoguardrails.colang.v2_x.lang.parser import parse_colang_file + + action = get_first_bot_action(("bot action: " + completion).splitlines()) + + assert action == expected + parsed = parse_colang_file("", f"flow generated\n {action}") + assert len(parsed["flows"]) == 1 + + +def test_generated_action_stops_before_next_dialogue_turn(): + action = get_first_bot_action( + "bot intent: bot greet\nbot action: bot say “Hello”\n" + 'user action: user said "Bye"\nbot intent: bot farewell\nbot action: bot say "Bye"'.splitlines() + ) + assert action == 'bot say "Hello"' + + class TestHasUnclosedQuote: """Tests for the _has_unclosed_quote helper.""" diff --git a/tests/recorded/rails/public_api/cassettes/test_dialog/test_dialog_generate_async_public_contract.yaml b/tests/recorded/rails/public_api/cassettes/test_dialog/test_dialog_generate_async_public_contract.yaml index 8e6d167c1e..f72015eee2 100644 --- a/tests/recorded/rails/public_api/cassettes/test_dialog/test_dialog_generate_async_public_contract.yaml +++ b/tests/recorded/rails/public_api/cassettes/test_dialog/test_dialog_generate_async_public_contract.yaml @@ -4,237 +4,34 @@ interactions: method: POST uri: https://api.openai.com/v1/chat/completions headers: - Host: + host: - api.openai.com - Accept: + accept: - '*/*' - Accept-Encoding: + accept-encoding: - gzip, deflate, zstd - Connection: + connection: - keep-alive - User-Agent: + user-agent: - python-httpx/0.28.1 - Content-Type: + content-type: - application/json parsed_body: model: gpt-5.4-nano messages: - - role: user - content: | - """ - Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ - - # This is how a conversation between a user and the bot can go: - User message: "Hello there!" - User intent: express greeting - Bot intent: express greeting - Bot message: "Hello! How can I assist you today?" - User message: "What can you do for me?" - User intent: ask about capabilities - Bot intent: respond about capabilities - Bot message: "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." - User message: "Tell me a bit about the history of NVIDIA." - User intent: ask general question - Bot intent: response for general question - Bot message: "NVIDIA is a technology company that specializes in designing and manufacturing graphics processing units (GPUs) and other computer hardware. The company was founded in 1993 by Jen-Hsun Huang, Chris Malachowsky, and Curtis Priem." - User message: "tell me more" - User intent: request more information - Bot intent: provide more information - Bot message: "Initially, the company focused on developing 3D graphics processing technology for the PC gaming market. In 1999, NVIDIA released the GeForce 256, the world's first GPU, which was a major breakthrough for the gaming industry. The company continued to innovate in the GPU space, releasing new products and expanding into other markets such as professional graphics, mobile devices, and artificial intelligence." - User message: "thanks" - User intent: express appreciation - Bot intent: express appreciation and offer additional help - Bot message: "You're welcome. If you have any more questions or if there's anything else I can help you with, please don't hesitate to ask." - - - # This is how the user talks: - User message: "hello" - User intent: express greeting - - - - # This is the current conversation between the user and the bot: - # Choose intent from this list: express greeting - User message: "Hello there!" - User intent: express greeting - Bot intent: express greeting - Bot message: "Hello! How can I assist you today?" - User message: "What can you do for me?" - User intent: ask about capabilities - Bot intent: respond about capabilities - Bot message: "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." - User message: "hello" - response: - status: - code: 200 - message: OK - headers: - Content-Type: - - application/json - Transfer-Encoding: - - chunked - Connection: - - keep-alive - Server: - - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Access-Control-Expose-Headers: - - CF-Ray - alt-svc: - - h3=":443"; ma=86400 - body: - parsed_body: - id: '[RECORDED_RESPONSE_ID]' - object: chat.completion - created: 0 - model: gpt-5.4-nano-2026-03-17 - choices: - - index: 0 - message: - role: assistant - content: Hello! How can I assist you today? - refusal: null - annotations: [] - finish_reason: stop - usage: - prompt_tokens: 568 - completion_tokens: 12 - total_tokens: 580 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: null -- request: - method: POST - uri: https://api.openai.com/v1/chat/completions - headers: - Host: - - api.openai.com - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate, zstd - Connection: - - keep-alive - User-Agent: - - python-httpx/0.28.1 - Content-Type: - - application/json - parsed_body: - model: gpt-5.4-nano - messages: - - role: user - content: | - """ + - role: system + content: |- Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ - - # This is how a conversation between a user and the bot can go: - User intent: express greeting - Bot intent: express greeting - User intent: ask about capabilities - Bot intent: respond about capabilities - User intent: ask general question - Bot intent: response for general question - User intent: request more information - Bot intent: provide more information - User intent: express appreciation - Bot intent: express appreciation and offer additional help - - # This is how the bot thinks: - User intent: express greeting - Bot intent: express greeting - - - - # This is the current conversation between the user and the bot: - User intent: express greeting - Bot intent: express greeting - User intent: ask about capabilities - Bot intent: respond about capabilities - User intent: Hello! How can I assist you today? - response: - status: - code: 200 - message: OK - headers: - Content-Type: - - application/json - Transfer-Encoding: - - chunked - Connection: - - keep-alive - Server: - - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Access-Control-Expose-Headers: - - CF-Ray - alt-svc: - - h3=":443"; ma=86400 - body: - parsed_body: - id: '[RECORDED_RESPONSE_ID]' - object: chat.completion - created: 0 - model: gpt-5.4-nano-2026-03-17 - choices: - - index: 0 - message: - role: assistant - content: Hi! 👋 How can I help you today? - refusal: null - annotations: [] - finish_reason: stop - usage: - prompt_tokens: 228 - completion_tokens: 14 - total_tokens: 242 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: null -- request: - method: POST - uri: https://api.openai.com/v1/chat/completions - headers: - Host: - - api.openai.com - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate, zstd - Connection: - - keep-alive - User-Agent: - - python-httpx/0.28.1 - Content-Type: - - application/json - parsed_body: - model: gpt-5.4-nano - messages: + Classify the latest user message into a Colang 1 user intent. + Prefer a matching intent from the examples. If none matches, generate + a short intent name that describes the user's request. + Return exactly one line: User intent: + Do not answer the user, repeat their message, or include explanations + or Markdown fences. - role: user content: | - """ - Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ - - # This is how a conversation between a user and the bot can go: + Example conversation: User message: "Hello there!" User intent: express greeting Bot intent: express greeting @@ -257,54 +54,34 @@ interactions: Bot message: "You're welcome. If you have any more questions or if there's anything else I can help you with, please don't hesitate to ask." + Intent examples: + User message: "hello" + User intent: express greeting - # This is how the bot talks: - Bot intent: inform cannot engage with sensitive content - Bot message: "I will not engage with sensitive content." - - Bot intent: inform cannot engage with inappropriate content - Bot message: "I will not engage with inappropriate content." - - Bot intent: inform cannot engage in self harm behavior - Bot message: "I will not engage in any self harm behavior." - - Bot intent: inform cannot engage in abusive or harmful behavior - Bot message: "I will not engage in any abusive or harmful behavior." - - Bot intent: express greeting - Bot message: "Hello from recorded dialog." - + Candidate intents: express greeting - # This is the current conversation between the user and the bot: - User message: "Hello there!" - User intent: express greeting - Bot intent: express greeting - Bot message: "Hello! How can I assist you today?" - User message: "What can you do for me?" - User intent: ask about capabilities - Bot intent: respond about capabilities - Bot message: "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." + Conversation: User message: "hello" - User intent: Hello! How can I assist you today? - Bot intent: general response response: status: code: 200 message: OK headers: - Content-Type: + content-type: - application/json - Transfer-Encoding: + transfer-encoding: - chunked - Connection: + connection: - keep-alive - Server: + server: - cloudflare - Strict-Transport-Security: + strict-transport-security: - max-age=31536000; includeSubDomains; preload - Access-Control-Expose-Headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray - CF-Ray alt-svc: - h3=":443"; ma=86400 @@ -318,14 +95,14 @@ interactions: - index: 0 message: role: assistant - content: Hello! How can I assist you today? + content: 'User intent: express greeting' refusal: null annotations: [] finish_reason: stop usage: - prompt_tokens: 677 - completion_tokens: 12 - total_tokens: 689 + prompt_tokens: 509 + completion_tokens: 8 + total_tokens: 517 prompt_tokens_details: cached_tokens: 0 audio_tokens: 0 diff --git a/tests/recorded/rails/public_api/cassettes/test_dialog/test_dialog_generation_bot_message_public_contract.yaml b/tests/recorded/rails/public_api/cassettes/test_dialog/test_dialog_generation_bot_message_public_contract.yaml index f8a4f56fbc..fe597eb908 100644 --- a/tests/recorded/rails/public_api/cassettes/test_dialog/test_dialog_generation_bot_message_public_contract.yaml +++ b/tests/recorded/rails/public_api/cassettes/test_dialog/test_dialog_generation_bot_message_public_contract.yaml @@ -19,13 +19,19 @@ interactions: parsed_body: model: gpt-5.4-nano messages: - - role: user - content: | - """ + - role: system + content: |- Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ - # This is how a conversation between a user and the bot can go: + Classify the latest user message into a Colang 1 user intent. + Prefer a matching intent from the examples. If none matches, generate + a short intent name that describes the user's request. + Return exactly one line: User intent: + Do not answer the user, repeat their message, or include explanations + or Markdown fences. + - role: user + content: | + Example conversation: User message: "Hello there!" User intent: express greeting Bot intent: express greeting @@ -48,22 +54,15 @@ interactions: Bot message: "You're welcome. If you have any more questions or if there's anything else I can help you with, please don't hesitate to ask." - # This is how the user talks: + Intent examples: User message: "tell me about your product" User intent: ask about the product - # This is the current conversation between the user and the bot: - # Choose intent from this list: ask about the product - User message: "Hello there!" - User intent: express greeting - Bot intent: express greeting - Bot message: "Hello! How can I assist you today?" - User message: "What can you do for me?" - User intent: ask about capabilities - Bot intent: respond about capabilities - Bot message: "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." + Candidate intents: ask about the product + + Conversation: User message: "tell me about your product" response: status: @@ -96,23 +95,14 @@ interactions: - index: 0 message: role: assistant - content: |- - Hello! I’m an AI assistant designed to help with a wide range of tasks through conversation. Here are a few things I can do that often feel “product-like”: - - - **Answer questions**: Explain topics, help with general knowledge, and walk through concepts. - - **Generate text**: Draft emails, messages, summaries, stories, posts, or structured documents. - - **Help with writing & editing**: Improve clarity, tone, grammar, and rewrite content for different audiences. - - **Assist with problem-solving**: Break down steps for coding, math, study plans, brainstorming, and troubleshooting (as long as the details you provide are enough). - - **Support planning & ideation**: Generate ideas, outlines, checklists, and options for decisions. - - If you tell me what you want to accomplish (e.g., writing something, learning a topic, coding help, planning), I can tailor the assistance to that goal. What kind of task are you thinking about? + content: 'User intent: ask about the product' refusal: null annotations: [] finish_reason: stop usage: - prompt_tokens: 580 - completion_tokens: 200 - total_tokens: 780 + prompt_tokens: 521 + completion_tokens: 10 + total_tokens: 531 prompt_tokens_details: cached_tokens: 0 audio_tokens: 0 @@ -142,127 +132,17 @@ interactions: parsed_body: model: gpt-5.4-nano messages: - - role: user - content: | - """ + - role: system + content: |- Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ - # This is how a conversation between a user and the bot can go: - User intent: express greeting - Bot intent: express greeting - User intent: ask about capabilities - Bot intent: respond about capabilities - User intent: ask general question - Bot intent: response for general question - User intent: request more information - Bot intent: provide more information - User intent: express appreciation - Bot intent: express appreciation and offer additional help - - - # This is how the bot thinks: - User intent: ask about the product - Bot intent: describe the product - - User intent: express greeting - Bot intent: express greeting - - - - # This is the current conversation between the user and the bot: - User intent: express greeting - Bot intent: express greeting - User intent: ask about capabilities - Bot intent: respond about capabilities - User intent: Hello! I’m an AI assistant designed to help with a wide range of tasks through conversation. Here are a few things I can do that often feel “product-like”: - response: - status: - code: 200 - message: OK - headers: - content-type: - - application/json - transfer-encoding: - - chunked - connection: - - keep-alive - server: - - cloudflare - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - access-control-expose-headers: - - X-Request-ID - - CF-Ray - - CF-Ray - alt-svc: - - h3=":443"; ma=86400 - body: - parsed_body: - id: '[RECORDED_RESPONSE_ID]' - object: chat.completion - created: 0 - model: gpt-5.4-nano-2026-03-17 - choices: - - index: 0 - message: - role: assistant - content: |- - Hello! I’m here to help with a wide range of tasks through conversation. A few things I can do that often feel “product-like” include: - - - **Answer questions and explain concepts** (from quick facts to deeper walkthroughs) - - **Help you write and edit** (emails, essays, resumes, cover letters, posts, etc.) - - **Brainstorm ideas** (projects, names, outlines, strategies, content themes) - - **Summarize and extract key points** from text you provide - - **Assist with planning** (study plans, travel itineraries, schedules, checklists) - - **Draft and refine messages** in a particular tone (friendly, formal, persuasive, etc.) - - **Provide coding help** (debugging, writing snippets, explaining errors, algorithm ideas) - - **Role-play or simulate conversations** (interviews, customer support, practice dialogues) - - If you tell me what you’re trying to do today, I can jump in right away. What would you like help with? - refusal: null - annotations: [] - finish_reason: stop - usage: - prompt_tokens: 269 - completion_tokens: 208 - total_tokens: 477 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: null -- request: - method: POST - uri: https://api.openai.com/v1/chat/completions - headers: - host: - - api.openai.com - accept: - - '*/*' - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - user-agent: - - python-httpx/0.28.1 - content-type: - - application/json - parsed_body: - model: gpt-5.4-nano - messages: + Write the spoken response for the latest bot intent in the conversation. + Return only: Bot message: "" + Use straight double quotes around the response. Do not repeat intents, + invent another turn, or include explanations or Markdown fences. - role: user content: | - """ - Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ - - # This is how a conversation between a user and the bot can go: + Example conversation: User message: "Hello there!" User intent: express greeting Bot intent: express greeting @@ -285,16 +165,7 @@ interactions: Bot message: "You're welcome. If you have any more questions or if there's anything else I can help you with, please don't hesitate to ask." - - # This is some additional context: - ```markdown - The product is a guardrails toolkit for large language models. It lets - developers add programmable rails that check user input, steer the dialog, and - validate bot output before it reaches the user. - ``` - - - # This is how the bot talks: + Bot response examples: Bot intent: inform cannot engage with inappropriate content Bot message: "I will not engage with inappropriate content." @@ -312,18 +183,17 @@ interactions: - # This is the current conversation between the user and the bot: - User message: "Hello there!" - User intent: express greeting - Bot intent: express greeting - Bot message: "Hello! How can I assist you today?" - User message: "What can you do for me?" - User intent: ask about capabilities - Bot intent: respond about capabilities - Bot message: "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." + + Reference information: + The product is a guardrails toolkit for large language models. It lets + developers add programmable rails that check user input, steer the dialog, and + validate bot output before it reaches the user. + + + Conversation: User message: "tell me about your product" - User intent: Hello! I’m an AI assistant designed to help with a wide range of tasks through conversation. Here are a few things I can do that often feel “product-like”: - Bot intent: general response + User intent: ask about the product + Bot intent: describe the product response: status: code: 200 @@ -355,23 +225,17 @@ interactions: - index: 0 message: role: assistant - content: |- - Sure! If you mean the **guardrails toolkit for large language models** you mentioned in the context, here’s what it is and what it helps you do: - - - **Programmable “rails” for LLMs:** Lets developers define rules and logic that run during the conversation. - - **Check user input:** You can validate or filter what users send (e.g., detect disallowed content, enforce formatting, constrain requests). - - **Steer the dialog:** You can route or modify the assistant’s behavior based on the input or conversation state (e.g., ask clarifying questions, refuse safely, switch modes). - - **Validate bot output:** Before the response is shown to the user, rails can verify it meets requirements (e.g., must include/avoid certain content, follow a policy, adhere to a schema). - - **Reduce unsafe or off-policy responses:** The overall goal is to make LLM behavior more controlled, consistent, and safer. - - If you tell me what kind of use case you’re building (customer support, tutoring, legal/health Q&A, internal tools, etc.), I can suggest what kinds of rails you’d typically implement. + content: 'Bot message: "My product is a guardrails toolkit for large language models. It helps developers add + programmable “rails” that can check and validate what users say, steer the conversation flow, and verify or + filter the bot’s output before it’s sent back to the user. The goal is to make LLM applications safer, more + reliable, and easier to control—without needing to hard-code every rule into the model itself."' refusal: null annotations: [] finish_reason: stop usage: - prompt_tokens: 753 - completion_tokens: 235 - total_tokens: 988 + prompt_tokens: 637 + completion_tokens: 88 + total_tokens: 725 prompt_tokens_details: cached_tokens: 0 audio_tokens: 0 diff --git a/tests/recorded/rails/public_api/cassettes/test_dialog/test_single_call_generate_async_public_contract.yaml b/tests/recorded/rails/public_api/cassettes/test_dialog/test_single_call_generate_async_public_contract.yaml index ecb962f08b..b44b3866cc 100644 --- a/tests/recorded/rails/public_api/cassettes/test_dialog/test_single_call_generate_async_public_contract.yaml +++ b/tests/recorded/rails/public_api/cassettes/test_dialog/test_single_call_generate_async_public_contract.yaml @@ -4,85 +4,92 @@ interactions: method: POST uri: https://api.openai.com/v1/chat/completions headers: - Host: + host: - api.openai.com - Accept: + accept: - '*/*' - Accept-Encoding: + accept-encoding: - gzip, deflate, zstd - Connection: + connection: - keep-alive - User-Agent: + user-agent: - python-httpx/0.28.1 - Content-Type: + content-type: - application/json parsed_body: model: gpt-5.4-nano messages: + - role: system + content: |- + Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. + + Generate the next turn of a Colang 1 interaction, using exactly this format: + User intent: + Bot intent: + Bot message: "" + + Prefer a matching user intent from the examples. If none matches, + generate a short intent name that describes the user's request. + Use straight double quotes around the response. Generate only these + two intents and the bot message. Do not repeat the user message, + invent another turn, or include explanations or Markdown fences. - role: user content: | - """ - Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ + Example conversation: + User message: "Hello there!" + User intent: express greeting + Bot intent: express greeting + Bot message: "Hello! How can I assist you today?" + User message: "What can you do for me?" + User intent: ask about capabilities + Bot intent: respond about capabilities + Bot message: "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." + User message: "Tell me a bit about the history of NVIDIA." + User intent: ask general question + Bot intent: response for general question + Bot message: "NVIDIA is a technology company that specializes in designing and manufacturing graphics processing units (GPUs) and other computer hardware. The company was founded in 1993 by Jen-Hsun Huang, Chris Malachowsky, and Curtis Priem." + User message: "tell me more" + User intent: request more information + Bot intent: provide more information + Bot message: "Initially, the company focused on developing 3D graphics processing technology for the PC gaming market. In 1999, NVIDIA released the GeForce 256, the world's first GPU, which was a major breakthrough for the gaming industry. The company continued to innovate in the GPU space, releasing new products and expanding into other markets such as professional graphics, mobile devices, and artificial intelligence." + User message: "thanks" + User intent: express appreciation + Bot intent: express appreciation and offer additional help + Bot message: "You're welcome. If you have any more questions or if there's anything else I can help you with, please don't hesitate to ask." - # This is how a conversation between a user and the bot can go: - user "Hello there!" - express greeting - bot express greeting - "Hello! How can I assist you today?" - user "What can you do for me?" - ask about capabilities - bot respond about capabilities - "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." - user "Tell me a bit about the history of NVIDIA." - ask general question - bot response for general question - "NVIDIA is a technology company that specializes in designing and manufacturing graphics processing units (GPUs) and other computer hardware. The company was founded in 1993 by Jen-Hsun Huang, Chris Malachowsky, and Curtis Priem." - user "tell me more" - request more information - bot provide more information - "Initially, the company focused on developing 3D graphics processing technology for the PC gaming market. In 1999, NVIDIA released the GeForce 256, the world's first GPU, which was a major breakthrough for the gaming industry. The company continued to innovate in the GPU space, releasing new products and expanding into other markets such as professional graphics, mobile devices, and artificial intelligence." - user "thanks" - express appreciation - bot express appreciation and offer additional help - "You're welcome. If you have any more questions or if there's anything else I can help you with, please don't hesitate to ask." + Flow examples: + User message: "hello" + User intent: express greeting + Bot intent: express greeting + Bot message: "Hello from recorded single call." - # For each user message, generate the next steps and finish with the bot message. - # These are some examples how the bot thinks: - user "hello" - express greeting - bot express greeting - "Hello from recorded single call." + Candidate intents: express greeting - # This is the current conversation between the user and the bot: - user "Hello there!" - express greeting - bot express greeting - "Hello! How can I assist you today?" - user "What can you do for me?" - ask about capabilities - bot respond about capabilities - "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." - user "hello" + + + Conversation: + User message: "hello" response: status: code: 200 message: OK headers: - Content-Type: + content-type: - application/json - Transfer-Encoding: + transfer-encoding: - chunked - Connection: + connection: - keep-alive - Server: + server: - cloudflare - Strict-Transport-Security: + strict-transport-security: - max-age=31536000; includeSubDomains; preload - Access-Control-Expose-Headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray - CF-Ray alt-svc: - h3=":443"; ma=86400 @@ -97,17 +104,16 @@ interactions: message: role: assistant content: |- - user "hello" - express greeting - bot express greeting - "Hello again! 😊 How can I assist you today?" + User intent: express greeting + Bot intent: express greeting + Bot message: "Hello from recorded single call." refusal: null annotations: [] finish_reason: stop usage: - prompt_tokens: 528 - completion_tokens: 28 - total_tokens: 556 + prompt_tokens: 556 + completion_tokens: 25 + total_tokens: 581 prompt_tokens_details: cached_tokens: 0 audio_tokens: 0 diff --git a/tests/recorded/rails/public_api/cassettes/test_requests/test_task_specific_models_generate_async.yaml b/tests/recorded/rails/public_api/cassettes/test_requests/test_task_specific_models_generate_async.yaml index 303569b0a2..f72015eee2 100644 --- a/tests/recorded/rails/public_api/cassettes/test_requests/test_task_specific_models_generate_async.yaml +++ b/tests/recorded/rails/public_api/cassettes/test_requests/test_task_specific_models_generate_async.yaml @@ -4,237 +4,34 @@ interactions: method: POST uri: https://api.openai.com/v1/chat/completions headers: - Host: + host: - api.openai.com - Accept: + accept: - '*/*' - Accept-Encoding: + accept-encoding: - gzip, deflate, zstd - Connection: + connection: - keep-alive - User-Agent: + user-agent: - python-httpx/0.28.1 - Content-Type: + content-type: - application/json parsed_body: model: gpt-5.4-nano messages: - - role: user - content: | - """ - Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ - - # This is how a conversation between a user and the bot can go: - User message: "Hello there!" - User intent: express greeting - Bot intent: express greeting - Bot message: "Hello! How can I assist you today?" - User message: "What can you do for me?" - User intent: ask about capabilities - Bot intent: respond about capabilities - Bot message: "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." - User message: "Tell me a bit about the history of NVIDIA." - User intent: ask general question - Bot intent: response for general question - Bot message: "NVIDIA is a technology company that specializes in designing and manufacturing graphics processing units (GPUs) and other computer hardware. The company was founded in 1993 by Jen-Hsun Huang, Chris Malachowsky, and Curtis Priem." - User message: "tell me more" - User intent: request more information - Bot intent: provide more information - Bot message: "Initially, the company focused on developing 3D graphics processing technology for the PC gaming market. In 1999, NVIDIA released the GeForce 256, the world's first GPU, which was a major breakthrough for the gaming industry. The company continued to innovate in the GPU space, releasing new products and expanding into other markets such as professional graphics, mobile devices, and artificial intelligence." - User message: "thanks" - User intent: express appreciation - Bot intent: express appreciation and offer additional help - Bot message: "You're welcome. If you have any more questions or if there's anything else I can help you with, please don't hesitate to ask." - - - # This is how the user talks: - User message: "hello" - User intent: express greeting - - - - # This is the current conversation between the user and the bot: - # Choose intent from this list: express greeting - User message: "Hello there!" - User intent: express greeting - Bot intent: express greeting - Bot message: "Hello! How can I assist you today?" - User message: "What can you do for me?" - User intent: ask about capabilities - Bot intent: respond about capabilities - Bot message: "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." - User message: "hello" - response: - status: - code: 200 - message: OK - headers: - Content-Type: - - application/json - Transfer-Encoding: - - chunked - Connection: - - keep-alive - Server: - - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Access-Control-Expose-Headers: - - CF-Ray - alt-svc: - - h3=":443"; ma=86400 - body: - parsed_body: - id: '[RECORDED_RESPONSE_ID]' - object: chat.completion - created: 0 - model: gpt-5.4-nano-2026-03-17 - choices: - - index: 0 - message: - role: assistant - content: Hello! How can I assist you today? - refusal: null - annotations: [] - finish_reason: stop - usage: - prompt_tokens: 568 - completion_tokens: 12 - total_tokens: 580 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: null -- request: - method: POST - uri: https://api.openai.com/v1/chat/completions - headers: - Host: - - api.openai.com - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate, zstd - Connection: - - keep-alive - User-Agent: - - python-httpx/0.28.1 - Content-Type: - - application/json - parsed_body: - model: gpt-5.4-nano - messages: - - role: user - content: | - """ + - role: system + content: |- Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ - - # This is how a conversation between a user and the bot can go: - User intent: express greeting - Bot intent: express greeting - User intent: ask about capabilities - Bot intent: respond about capabilities - User intent: ask general question - Bot intent: response for general question - User intent: request more information - Bot intent: provide more information - User intent: express appreciation - Bot intent: express appreciation and offer additional help - - # This is how the bot thinks: - User intent: express greeting - Bot intent: express greeting - - - - # This is the current conversation between the user and the bot: - User intent: express greeting - Bot intent: express greeting - User intent: ask about capabilities - Bot intent: respond about capabilities - User intent: Hello! How can I assist you today? - response: - status: - code: 200 - message: OK - headers: - Content-Type: - - application/json - Transfer-Encoding: - - chunked - Connection: - - keep-alive - Server: - - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Access-Control-Expose-Headers: - - CF-Ray - alt-svc: - - h3=":443"; ma=86400 - body: - parsed_body: - id: '[RECORDED_RESPONSE_ID]' - object: chat.completion - created: 0 - model: gpt-5.4-nano-2026-03-17 - choices: - - index: 0 - message: - role: assistant - content: Hello! 👋 How can I assist you today? - refusal: null - annotations: [] - finish_reason: stop - usage: - prompt_tokens: 228 - completion_tokens: 14 - total_tokens: 242 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: null -- request: - method: POST - uri: https://api.openai.com/v1/chat/completions - headers: - Host: - - api.openai.com - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate, zstd - Connection: - - keep-alive - User-Agent: - - python-httpx/0.28.1 - Content-Type: - - application/json - parsed_body: - model: gpt-5.4-nano - messages: + Classify the latest user message into a Colang 1 user intent. + Prefer a matching intent from the examples. If none matches, generate + a short intent name that describes the user's request. + Return exactly one line: User intent: + Do not answer the user, repeat their message, or include explanations + or Markdown fences. - role: user content: | - """ - Below is a conversation between a helpful AI assistant and a user. The bot is designed to generate human-like text based on the input that it receives. The bot is talkative and provides lots of specific details. If the bot does not know the answer to a question, it truthfully says it does not know. - """ - - # This is how a conversation between a user and the bot can go: + Example conversation: User message: "Hello there!" User intent: express greeting Bot intent: express greeting @@ -257,54 +54,34 @@ interactions: Bot message: "You're welcome. If you have any more questions or if there's anything else I can help you with, please don't hesitate to ask." + Intent examples: + User message: "hello" + User intent: express greeting - # This is how the bot talks: - Bot intent: inform cannot engage with sensitive content - Bot message: "I will not engage with sensitive content." - - Bot intent: inform cannot engage with inappropriate content - Bot message: "I will not engage with inappropriate content." - - Bot intent: inform cannot engage in self harm behavior - Bot message: "I will not engage in any self harm behavior." - - Bot intent: inform cannot engage in abusive or harmful behavior - Bot message: "I will not engage in any abusive or harmful behavior." - - Bot intent: express greeting - Bot message: "Hello from recorded task models." - + Candidate intents: express greeting - # This is the current conversation between the user and the bot: - User message: "Hello there!" - User intent: express greeting - Bot intent: express greeting - Bot message: "Hello! How can I assist you today?" - User message: "What can you do for me?" - User intent: ask about capabilities - Bot intent: respond about capabilities - Bot message: "As an AI assistant, I can help you with a wide range of tasks. This includes question answering on various topics, generating text for various purposes and providing suggestions based on your preferences." + Conversation: User message: "hello" - User intent: Hello! How can I assist you today? - Bot intent: general response response: status: code: 200 message: OK headers: - Content-Type: + content-type: - application/json - Transfer-Encoding: + transfer-encoding: - chunked - Connection: + connection: - keep-alive - Server: + server: - cloudflare - Strict-Transport-Security: + strict-transport-security: - max-age=31536000; includeSubDomains; preload - Access-Control-Expose-Headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray - CF-Ray alt-svc: - h3=":443"; ma=86400 @@ -318,14 +95,14 @@ interactions: - index: 0 message: role: assistant - content: Hi! 👋 How can I help you today? + content: 'User intent: express greeting' refusal: null annotations: [] finish_reason: stop usage: - prompt_tokens: 678 - completion_tokens: 14 - total_tokens: 692 + prompt_tokens: 509 + completion_tokens: 8 + total_tokens: 517 prompt_tokens_details: cached_tokens: 0 audio_tokens: 0 diff --git a/tests/recorded/rails/public_api/test_dialog.py b/tests/recorded/rails/public_api/test_dialog.py index 931e5a1407..da8887e60e 100644 --- a/tests/recorded/rails/public_api/test_dialog.py +++ b/tests/recorded/rails/public_api/test_dialog.py @@ -38,24 +38,18 @@ async def test_dialog_generate_async_public_contract(openai_api_key): assert result == snapshot( { "role": "assistant", - "content": "Hello! How can I assist you today?", + "content": "Hello from recorded dialog.", } ) @pytest.mark.vcr async def test_dialog_generation_bot_message_public_contract(openai_api_key): - """Bot-message GENERATION prompt carries few-shot examples + retrieved kb chunks. - - "tell me about your product" maps (via a flow) to the bot intent - "describe the product", which has no predefined message, so - ``generate_bot_message`` calls the LLM with the bot-message index examples and - the retrieved kb chunks. VCR matches on ``recorded_body``, so this cassette - pins all three dialog prompts (intent, next-step, bot-message) by their exact - request bodies. It was verified to replay byte-identically against both the - pre-refactor ``generation.py`` on ``develop`` and the refactored branch, so - the prompts are unchanged pre/post -- a prompt-fidelity equivalence check the - prompt-independent ``FakeLLMModel`` oracle cannot make. + """Exercise intent classification and generated bot text with retrieved context. + + The product question selects a flow whose bot intent has no predefined + message. Its generation prompt includes bot-message examples and knowledge + base chunks. VCR matches the complete intent and bot-message request bodies. """ rails = LLMRails(load_config(DIALOG_GENERATION_CONFIG), verbose=False) @@ -65,15 +59,7 @@ async def test_dialog_generation_bot_message_public_contract(openai_api_key): assert result == snapshot( { "role": "assistant", - "content": """\ -Sure! If you mean the **guardrails toolkit for large language models** you mentioned in the context, here’s what it is and what it helps you do: -- **Programmable “rails” for LLMs:** Lets developers define rules and logic that run during the conversation. -- **Check user input:** You can validate or filter what users send (e.g., detect disallowed content, enforce formatting, constrain requests). -- **Steer the dialog:** You can route or modify the assistant’s behavior based on the input or conversation state (e.g., ask clarifying questions, refuse safely, switch modes). -- **Validate bot output:** Before the response is shown to the user, rails can verify it meets requirements (e.g., must include/avoid certain content, follow a policy, adhere to a schema). -- **Reduce unsafe or off-policy responses:** The overall goal is to make LLM behavior more controlled, consistent, and safer. -If you tell me what kind of use case you’re building (customer support, tutoring, legal/health Q&A, internal tools, etc.), I can suggest what kinds of rails you’d typically implement.\ -""", + "content": "My product is a guardrails toolkit for large language models. It helps developers add programmable “rails” that can check and validate what users say, steer the conversation flow, and verify or filter the bot’s output before it’s sent back to the user. The goal is to make LLM applications safer, more reliable, and easier to control—without needing to hard-code every rule into the model itself.", } ) @@ -88,10 +74,7 @@ async def test_single_call_generate_async_public_contract(openai_api_key): assert result == snapshot( { "role": "assistant", - "content": """\ -bot express greeting -"Hello again! 😊 How can I assist you today?"\ -""", + "content": "Hello from recorded single call.", } ) diff --git a/tests/recorded/rails/public_api/test_requests.py b/tests/recorded/rails/public_api/test_requests.py index 4391abee0b..32b0a61605 100644 --- a/tests/recorded/rails/public_api/test_requests.py +++ b/tests/recorded/rails/public_api/test_requests.py @@ -172,36 +172,18 @@ async def test_task_specific_models_generate_async(openai_api_key, record_mode, assert_request_payload(recorded_cassette_path, model=OPENAI_MODEL) assert normalize_generation_response(result) == snapshot( { - "response": [{"role": "assistant", "content": "Hi! 👋 How can I help you today?"}], + "response": [{"role": "assistant", "content": "Hello from recorded task models."}], "activated_rails": [], "llm_calls": [ { "task": "generate_user_intent", "provider": "openai", "model": "gpt-5.4-nano", - "completion": "Hello! How can I assist you today?", - "prompt_tokens": 568, - "completion_tokens": 12, - "total_tokens": 580, - }, - { - "task": "generate_next_steps", - "provider": "openai", - "model": "gpt-5.4-nano", - "completion": "Hello! 👋 How can I assist you today?", - "prompt_tokens": 228, - "completion_tokens": 14, - "total_tokens": 242, - }, - { - "task": "generate_bot_message", - "provider": "openai", - "model": "gpt-5.4-nano", - "completion": "Hi! 👋 How can I help you today?", - "prompt_tokens": 678, - "completion_tokens": 14, - "total_tokens": 692, - }, + "completion": "User intent: express greeting", + "prompt_tokens": 509, + "completion_tokens": 8, + "total_tokens": 517, + } ], } ) diff --git a/tests/test_current_openai_generation.py b/tests/test_current_openai_generation.py new file mode 100644 index 0000000000..25ef04996c --- /dev/null +++ b/tests/test_current_openai_generation.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import pytest + +from nemoguardrails import LLMRails, RailsConfig +from nemoguardrails.llm.prompts import get_prompt +from nemoguardrails.llm.taskmanager import LLMTaskManager +from nemoguardrails.rails.llm.options import GenerationResponse +from nemoguardrails.testing.fake_model import FakeLLMModel + + +@pytest.mark.parametrize("model", ["gpt-5.6-terra", "gpt-6-luna"]) +@pytest.mark.parametrize( + "task", + [ + "generate_user_intent", + "generate_next_steps", + "generate_bot_message", + "generate_intent_steps_message", + "generate_value", + ], +) +def test_current_models_use_colang1_chat_prompts(model, task): + config = RailsConfig.from_content(config={"models": [{"type": "main", "engine": "openai", "model": model}]}) + prompt = get_prompt(config, task) + assert prompt.messages + assert not prompt.content + rendered = LLMTaskManager(config).render_task_prompt( + task, + events=[{"type": "UserMessage", "text": "How much vacation?"}], + context={ + "examples": 'user "How much vacation?"\n ask about benefits\nbot explain benefits\n "Two weeks."', + "potential_user_intents": "ask about benefits", + "relevant_chunks": "Vacation policy: two weeks.", + "var_name": "is_safe", + "instructions": "Return True if safe, False otherwise.", + }, + ) + assert isinstance(rendered, list) + assert rendered[0]["type"] == "system" + text = "\n".join(message["content"] for message in rendered) + assert "How much vacation?" in text + if task in ("generate_bot_message", "generate_intent_steps_message"): + assert "Vacation policy: two weeks." in text + if task in ("generate_user_intent", "generate_intent_steps_message"): + assert "User intent: " in text + assert "ask about benefits" in text + if task == "generate_value": + assert "Python literal" in text + assert "Return True if safe, False otherwise." in text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("single_call", [False, True]) +async def test_sample_example_uses_greeting_flow(single_call): + config = RailsConfig.from_path(str(Path(__file__).parents[1] / "examples/configs/sample")) + config.models[0].model = "gpt-6-luna" + config.rails.dialog.single_call.enabled = single_call + answer = "Hello, hello! Glad you stopped by." + responses = ( + [f'User intent: express greeting\nBot intent: express greeting\nBot message: "{answer}"'] + if single_call + else ["User intent: express greeting", f'Bot message: "{answer}"'] + ) + model = FakeLLMModel(responses=responses) + rails = LLMRails(config, llm=model) + result = await rails.generate_async( + messages=[{"role": "user", "content": "hello"}], + options={"log": {"activated_rails": True, "internal_events": True}}, + ) + assert isinstance(result, GenerationResponse) + assert isinstance(result.response, list) + assert result.response[0]["content"] == answer + "!!" + assert result.log is not None and result.log.internal_events is not None + assert [ + (e["type"], e["intent"]) for e in result.log.internal_events if e["type"] in ("UserIntent", "BotIntent") + ] == [("UserIntent", "express greeting"), ("BotIntent", "express greeting")] + assert {rail.name for rail in result.log.activated_rails} >= {"dummy input rail", "dummy output rail"} + assert model.inference_count == (1 if single_call else 2) diff --git a/tests/test_llmrails_singlecall.py b/tests/test_llmrails_singlecall.py index c8255a7351..ddf6d6d581 100644 --- a/tests/test_llmrails_singlecall.py +++ b/tests/test_llmrails_singlecall.py @@ -19,6 +19,7 @@ import pytest from nemoguardrails import LLMRails, RailsConfig +from nemoguardrails.rails.llm.options import GenerationResponse from nemoguardrails.testing.fake_model import FakeLLMModel from nemoguardrails.types import LLMResponse, ToolCall, ToolCallChunk, ToolCallDeltaChunk from tests.utils import TestChat @@ -257,3 +258,67 @@ async def wait_for_completed_producer(self, k): chunks = await asyncio.wait_for(_collect_single_call_stream(rails), timeout=3) assert "".join(chunks) == expected assert model.inference_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("verbose", [False, True]) +@pytest.mark.parametrize("echo", [False, True]) +@pytest.mark.parametrize("finish_before_handoff", [False, True]) +async def test_single_call_handles_echoed_user_message(monkeypatch, streaming, verbose, echo, finish_before_handoff): + if finish_before_handoff: + from nemoguardrails.streaming import StreamingHandler + + original_wait = StreamingHandler.wait_top_k_nonempty_lines + + async def wait_for_completed_producer(self, k): + await self._producer_task + return await original_wait(self, k) + + monkeypatch.setattr(StreamingHandler, "wait_top_k_nonempty_lines", wait_for_completed_producer) + + question = "Tell me a short fun fact about the Moon." + answer = "Fun fact: The Moon drifts about 3.8 centimeters farther from Earth each year." + if verbose: + completion = f'User intent: ask for a fun fact\nBot intent: share fun fact\nBot message: "{answer}"' + echoed_message = f'User message: "{question}"\n' + else: + completion = f' ask for a fun fact\nbot share fun fact\n "{answer}"' + echoed_message = f'user "{question}"\n' + if echo: + completion = echoed_message + completion + config = RailsConfig.from_content( + config={ + "models": [], + "rails": {"dialog": {"single_call": {"enabled": True}}}, + "prompts": [ + { + "task": "generate_intent_steps_message", + "content": "Generate the next turn.", + "models": ["unknown"], + "output_parser": "verbose_v1" if verbose else None, + } + ], + }, + colang_content='define user express greeting\n "hello"\ndefine flow\n user express greeting\n bot express greeting\n', + ) + model = FakeLLMModel(responses=[completion]) + rails = LLMRails(config, llm=model) + messages = [{"role": "user", "content": question}] + if streaming: + + async def collect(): + return "".join([chunk async for chunk in rails.stream_async(messages=messages)]) + + content = await asyncio.wait_for(collect(), timeout=3) + else: + result = await rails.generate_async(messages=messages, options={"log": {"internal_events": True}}) + assert isinstance(result, GenerationResponse) + assert isinstance(result.response, list) + assert result.log is not None and result.log.internal_events is not None + content = result.response[0]["content"] + assert [ + (e["type"], e["intent"]) for e in result.log.internal_events if e["type"] in ("UserIntent", "BotIntent") + ] == [("UserIntent", "ask for a fun fact"), ("BotIntent", "share fun fact")] + assert content == answer + assert model.inference_count == 1 diff --git a/tests/test_streaming_handler.py b/tests/test_streaming_handler.py index 1cb7315c7c..d1948469f1 100644 --- a/tests/test_streaming_handler.py +++ b/tests/test_streaming_handler.py @@ -1087,3 +1087,50 @@ async def test_suffix_whitespace_preserves_message_content(body, expected): await handler.finish() assert "".join(await asyncio.wait_for(_collect_handler(handler), timeout=1)) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("producer_fails", [False, True]) +async def test_repeated_line_reads_wait_for_complete_line(producer_fails): + handler = StreamingHandler() + await handler.enable_buffering() + release = asyncio.Event() + + async def produce(): + await handler.push_chunk('user "hello"\nexpress greeting\nbot ') + await release.wait() + if producer_fails: + raise RuntimeError("provider interrupted") + await handler.push_chunk('express greeting\n "Hello!"') + await handler.finish() + + producer = asyncio.create_task(produce()) + handler.set_producer_task(producer) + first = await asyncio.wait_for(handler.wait_top_k_nonempty_lines(2), timeout=2) + assert first == 'user "hello"\nexpress greeting' + second = asyncio.create_task(handler.wait_top_k_nonempty_lines(1)) + try: + await asyncio.sleep(0) + assert not second.done() + release.set() + if producer_fails: + with pytest.raises(RuntimeError, match="provider interrupted"): + await asyncio.wait_for(second, timeout=2) + else: + assert await asyncio.wait_for(second, timeout=2) == "bot express greeting" + assert handler.buffer == ' "Hello!"' + finally: + producer.cancel() + second.cancel() + await asyncio.gather(producer, second, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_repeated_line_reads_after_buffered_end_of_stream(): + handler = StreamingHandler() + await handler.enable_buffering() + await handler.push_chunk("first\nsecond\nlast") + assert await asyncio.wait_for(handler.wait_top_k_nonempty_lines(2), timeout=2) == "first\nsecond" + await handler.finish() + assert await asyncio.wait_for(handler.wait_top_k_nonempty_lines(1), timeout=2) == "last" + assert await asyncio.wait_for(handler.wait_top_k_nonempty_lines(1), timeout=2) == "" diff --git a/tests/v2_x/test_current_openai_generation.py b/tests/v2_x/test_current_openai_generation.py new file mode 100644 index 0000000000..b4ae42193b --- /dev/null +++ b/tests/v2_x/test_current_openai_generation.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import pytest + +from nemoguardrails import RailsConfig +from nemoguardrails.llm.prompts import get_prompt +from nemoguardrails.llm.taskmanager import LLMTaskManager +from nemoguardrails.testing.chat_harness import TestChat + + +@pytest.mark.parametrize("model", ["gpt-6-luna", "gpt-5.6-terra"]) +@pytest.mark.parametrize( + "task", + [ + "generate_user_intent_from_user_action", + "generate_user_intent_and_bot_action_from_user_action", + "generate_flow_continuation", + "generate_value_from_instruction", + ], +) +def test_current_models_use_explicit_colang_chat_prompts(model, task): + config = RailsConfig.from_content( + yaml_content=f'colang_version: "2.x"\nmodels:\n - type: main\n engine: openai\n model: {model}' + ) + prompt = get_prompt(config, task) + assert prompt.messages + assert not prompt.content + rendered = LLMTaskManager(config).render_task_prompt( + task, + events=[], + context={ + "examples": "user ask about benefits", + "context": {"relevant_chunks": "Vacation policy: two weeks."}, + "user_action": 'user said "How much vacation?"', + "var_name": "is_safe", + "instructions": "Return True if safe, False otherwise.", + }, + ) + text = "\n".join(message["content"] for message in rendered) + if task in ("generate_flow_continuation", "generate_user_intent_and_bot_action_from_user_action"): + assert "bot intent:" in text + assert 'bot action: bot say ""' in text + assert "Vacation policy: two weeks." in text + elif task == "generate_value_from_instruction": + assert "Python literal" in text + assert "Return True if safe, False otherwise." in text + else: + assert "user intent:" in text + assert "user ask about benefits" in text + + +@pytest.mark.parametrize("example", ["bots/abc_v2", "v2_x/tutorial/guardrails_1"]) +def test_shipped_example_with_curly_quoted_continuation(example): + config = RailsConfig.from_path(str(Path(__file__).parents[2] / "examples" / example)) + config.models[0].model = "gpt-6-luna" + completion = ( + "bot intent: bot answer question\n" + "bot action: bot say “Employees get two weeks of paid vacation.”\n" + 'user action: user said "Thanks"\n' + 'bot intent: bot acknowledge\nbot action: bot say "You are welcome"' + ) + responses = ["user intent: user ask question about benefits", completion] + if example == "bots/abc_v2": + responses = ["No", *responses, "No"] + else: + responses = ["True", *responses] + chat = TestChat(config, llm_completions=responses) + + chat >> "How much paid vacation do employees get?" + chat << "Employees get two weeks of paid vacation." + assert chat.llm.inference_count == len(responses) diff --git a/tests/v2_x/test_generation_actions_unit.py b/tests/v2_x/test_generation_actions_unit.py index 03be69e445..29aabb0987 100644 --- a/tests/v2_x/test_generation_actions_unit.py +++ b/tests/v2_x/test_generation_actions_unit.py @@ -25,6 +25,7 @@ from nemoguardrails.colang.v2_x.runtime.flows import InternalEvents from nemoguardrails.context import generation_options_var, raw_llm_request from nemoguardrails.rails.llm.options import GenerationOptions +from nemoguardrails.testing.fake_model import FakeLLMModel class FakeIndex: @@ -327,6 +328,63 @@ async def test_generate_flow_continuation_success_fallback_and_error(monkeypatch await actions.generate_flow_continuation(state, []) +@pytest.mark.asyncio +@pytest.mark.parametrize("output", ["Here is your answer.", "bot intent: bot answer", "bot action: ", "bot action: "]) +@pytest.mark.parametrize("combined", [False, True]) +async def test_generation_rejects_missing_bot_action(output, combined): + actions = make_actions() + actions.llm = FakeLLMModel(responses=[output]) + actions.flows_index = FakeIndex() + actions.instruction_flows_index = FakeIndex() + actions._collect_user_intent_and_examples = AsyncMock(return_value=(["user ask"], "", False)) + state = SimpleNamespace(context={}) + + with pytest.raises(LlmResponseError): + if combined: + await actions.generate_user_intent_and_bot_action(state, [], "question") + else: + await actions.generate_flow_continuation(state, []) + + +@pytest.mark.asyncio +async def test_flow_continuation_indents_all_generated_actions(): + from nemoguardrails.colang.v2_x.lang.parser import parse_colang_file + + actions = make_actions() + actions.llm = FakeLLMModel( + responses=["bot intent: bot answer\nbot action: bot say “First”\nbot action: bot say “Second”"] + ) + actions.flows_index = FakeIndex() + actions.instruction_flows_index = FakeIndex() + + generated = await actions.generate_flow_continuation(SimpleNamespace(context={}), []) + + parsed = parse_colang_file("", generated["body"]) + assert len(parsed["flows"]) == 1 + assert '\n bot say "First"\n bot say "Second"' in generated["body"] + + +@pytest.mark.asyncio +async def test_combined_generation_creates_valid_multiaction_flow(): + from nemoguardrails.colang.v2_x.lang.parser import parse_colang_file + + actions = make_actions() + actions.llm = FakeLLMModel( + responses=[ + "user intent: user greet\nbot intent: bot greet\n" + 'bot action: bot say “Hello”\nbot action: bot gesture "wave"' + ] + ) + actions._collect_user_intent_and_examples = AsyncMock(return_value=(["user greet"], "", False)) + + result = await actions.generate_user_intent_and_bot_action(SimpleNamespace(context={}), [], "hi") + generated = await actions.create_flow([], result["bot_intent"], result["bot_action"]) + + parsed = parse_colang_file("", generated["body"]) + assert len(parsed["flows"]) == 1 + assert '\n bot say "Hello"\n bot gesture "wave"' in generated["body"] + + @pytest.mark.asyncio async def test_create_flow_escapes_name_and_applies_decorators(monkeypatch): actions = make_actions()