From 637c491a8b065028cea09ed66712dcdd99c8bb93 Mon Sep 17 00:00:00 2001 From: Razvan Dinu Date: Thu, 13 Jul 2023 12:52:41 +0300 Subject: [PATCH] Add event-based API using `generate_events`, `generate_events_async` and a new message type "event". --- CHANGELOG.md | 6 + docs/user_guide/advanced/event-based-api.md | 251 ++++++++++++++++++++ nemoguardrails/rails/llm/llmrails.py | 55 +++++ nemoguardrails/rails/llm/utils.py | 2 + tests/test_event_based_api.py | 122 ++++++++++ 5 files changed, 436 insertions(+) create mode 100644 docs/user_guide/advanced/event-based-api.md create mode 100644 tests/test_event_based_api.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f18a03a4..df51b751e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased + +### Added + +- [Event-based API](./docs/user_guide/advanced/event-based-api.md) for guardrails. +- Support for message with type "event" in [`LLMRails.generate_async`](./docs/api/nemoguardrails.rails.llm.llmrails.md#method-llmrailsgenerate_async). + ### Fixed - [#58](https://github.com/NVIDIA/NeMo-Guardrails/issues/58): Fix install on Mac OS 13. diff --git a/docs/user_guide/advanced/event-based-api.md b/docs/user_guide/advanced/event-based-api.md new file mode 100644 index 000000000..7bbf4c810 --- /dev/null +++ b/docs/user_guide/advanced/event-based-api.md @@ -0,0 +1,251 @@ +# Event-based API + +You can use a guardrails configuration through an event-based API using [`LLMRails.generate_events_async](../../api/nemoguardrails.rails.llm.llmrails.md#method-llmrailsgenerate_events_async) and [`LLMRails.generate_events](../../api/nemoguardrails.rails.llm.llmrails.md#method-llmrailsgenerate_events). + +Example usage: + +```python +import json +from nemoguardrails import LLMRails, RailsConfig + +config = RailsConfig.from_path("path/to/config") +app = LLMRails(config) + +new_events = app.generate_events(events=[{ + "type": "user_said", + "content": "Hello! What can you do for me?" +}]) +print(json.dumps(new_events, indent=True)) +``` + +Example output: +```yaml +[ + { + "type": "start_action", + "action_name": "generate_user_intent", + "action_params": {}, + "action_result_key": null, + "is_system_action": true + }, + { + "type": "action_finished", + "action_name": "generate_user_intent", + "action_params": {}, + "action_result_key": null, + "status": "success", + "return_value": null, + "events": [ + { + "type": "user_intent", + "intent": "express greeting" + } + ], + "is_system_action": true + }, + { + "type": "user_intent", + "intent": "express greeting" + }, + { + "type": "bot_intent", + "intent": "express greeting" + }, + { + "type": "start_action", + "action_name": "retrieve_relevant_chunks", + "action_params": {}, + "action_result_key": null, + "is_system_action": true + }, + { + "type": "context_update", + "data": { + "relevant_chunks": "" + } + }, + { + "type": "action_finished", + "action_name": "retrieve_relevant_chunks", + "action_params": {}, + "action_result_key": null, + "status": "success", + "return_value": "", + "events": null, + "is_system_action": true + }, + { + "type": "start_action", + "action_name": "generate_bot_message", + "action_params": {}, + "action_result_key": null, + "is_system_action": true + }, + { + "type": "context_update", + "data": { + "_last_bot_prompt": "<>>" + } + }, + { + "type": "action_finished", + "action_name": "generate_bot_message", + "action_params": {}, + "action_result_key": null, + "status": "success", + "return_value": null, + "events": [ + { + "type": "bot_said", + "content": "Hello!" + } + ], + "is_system_action": true + }, + { + "type": "bot_said", + "content": "Hello!" + }, + { + "type": "listen" + } +] +``` + +## Event Types + +NeMo Guardrails supports multiple types of events. Some are meant for internal use (e.g., `user_intent`, `bot_intent`), while others represent the "public" interface (e.g., `user_said`, `bot_said`). + +### `user_said` + +The raw message from the user. + +Example: +```json +{ + "type": "user_said", + "content": "Hello!" +} +``` + +### `user_intent` + +The computed intent (a.k.a. canonical form) for what the user said. + +Example: +```json +{ + "type": "user_intent", + "intent": "express greeting" +} +``` + +### `bot_intent` + +The computed intent for what the bot should say. + +Example: +```json +{ + "type": "bot_intent", + "intent": "express greeting" +} +``` + +### `bot_said` + +The final message from the bot. + +Example: +```json +{ + "type": "bot_said", + "content": "Hello!" +} +``` + +### `start_action` + +An action needs to be started. + +Example: +```json +{ + "type": "start_action", + "action_name": "generate_user_intent", + "action_params": {}, + "action_result_key": null, + "is_system_action": true +} +``` + +### `action_finished` + +An action has finished. + +Example: +```json +{ + "type": "action_finished", + "action_name": "generate_user_intent", + "action_params": {}, + "action_result_key": null, + "status": "success", + "return_value": null, + "events": [ + { + "type": "user_intent", + "intent": "express greeting" + } + ], + "is_system_action": true +} +``` + +### `context_update` + +The context of the conversation has been updated. + +Example: +```json +{ + "type": "context_update", + "data": { + "user_name": "John" + } +} +``` + +### `listen` + +The bot has finished processing the events and is waiting for new input. + +Example: +```json +{ + "type": "listen" +} +``` + +## Custom Events + +You can also use custom events: + +```json +{ + "type": "some_other_type", + ... +} +``` + +**Note**: You need to make sure that the guardrails logic can handle the custom event. + +## Typical Usage + +Typically, you will need to: + +1. Persist the events history for a particular user in a database. +2. Whenever there is a new message or another event, you fetch the history and append the new event. +3. Use the guardrails API to generate the next events. +4. Filter the `bot_said` events and return them to the user. +5. Persist the history of events back into the database. diff --git a/nemoguardrails/rails/llm/llmrails.py b/nemoguardrails/rails/llm/llmrails.py index 351ecbc7e..fa4756293 100644 --- a/nemoguardrails/rails/llm/llmrails.py +++ b/nemoguardrails/rails/llm/llmrails.py @@ -179,6 +179,8 @@ def _get_events_for_messages(self, messages: List[dict]): events.append({"type": "bot_said", "content": msg["content"]}) elif msg["role"] == "context": events.append({"type": "context_update", "data": msg["content"]}) + elif msg["role"] == "event": + events.append(msg["event"]) return events @@ -194,6 +196,7 @@ async def generate_async( {"role": "context", "content": {"user_name": "John"}}, {"role": "user", "content": "Hello! How are you?"}, {"role": "assistant", "content": "I am fine, thank you!"}, + {"role": "event", "event": {"type": "user_silent"}}, ... ] ``` @@ -273,6 +276,58 @@ def generate( return asyncio.run(self.generate_async(prompt=prompt, messages=messages)) + async def generate_events_async(self, events: List[dict]) -> List[dict]: + """Generate the next events based on the provided history. + + The format for events is the following: + + ```python + [ + {"type": "...", ...}, + ... + ] + ``` + + Args: + events: The history of events to be used to generate the next events. + + Returns: + The newly generate event(s). + + """ + t0 = time.time() + llm_stats.reset() + + # Compute the new events. + new_events = await self.runtime.generate_events(events) + + # If logging is enabled, we log the conversation + # TODO: add support for logging flag + if self.verbose: + history = get_colang_history(events) + log.info(f"Conversation history so far: \n{history}") + + log.info("--- :: Total processing took %.2f seconds." % (time.time() - t0)) + log.info("--- :: Stats: %s" % llm_stats) + + return new_events + + def generate_events(self, events: List[dict]) -> List[dict]: + """Synchronous version of `LLMRails.generate_events_async`.""" + + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + + if loop and loop.is_running(): + raise RuntimeError( + "You are using the sync `generate_events` inside async code. " + "You should replace with `await generate_events_async(...)." + ) + + return asyncio.run(self.generate_events_async(events=events)) + def register_action(self, action: callable, name: Optional[str] = None): """Register a custom action for the rails configuration.""" self.runtime.register_action(action, name) diff --git a/nemoguardrails/rails/llm/utils.py b/nemoguardrails/rails/llm/utils.py index 1ee1b40b4..c8a94e1d9 100644 --- a/nemoguardrails/rails/llm/utils.py +++ b/nemoguardrails/rails/llm/utils.py @@ -37,6 +37,8 @@ def get_history_cache_key(messages: List[dict]) -> str: key_items.append(msg["content"]) elif msg["role"] == "context": key_items.append(json.dumps(msg["content"])) + elif msg["role"] == "event": + key_items.append(json.dumps(msg["event"])) history_cache_key = ":".join(key_items) diff --git a/tests/test_event_based_api.py b/tests/test_event_based_api.py new file mode 100644 index 000000000..2b51844a6 --- /dev/null +++ b/tests/test_event_based_api.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +from nemoguardrails import RailsConfig +from tests.utils import TestChat + + +def test_1(): + config = RailsConfig.from_content( + """ + define user express greeting + "hello" + + define flow + user express greeting + bot express greeting + """ + ) + + chat = TestChat( + config, + llm_completions=[ + " express greeting", + ' "Hello!"', + ], + ) + + new_events = chat.app.generate_events( + events=[{"type": "user_said", "content": "Hello!"}] + ) + + # We don't want to pin the exact number here in the test as the exact number of events + # can vary as the implementation changes. + assert len(new_events) > 10 + + print(json.dumps(new_events, indent=True)) + + # We check certain key events are present. + assert {"intent": "express greeting", "type": "user_intent"} in new_events + assert {"intent": "express greeting", "type": "bot_intent"} in new_events + assert {"content": "Hello!", "type": "bot_said"} in new_events + assert {"type": "listen"} in new_events + + +CONFIG_WITH_EVENT = """ + define user express greeting + "hello" + + define flow + user express greeting + bot express greeting + + define flow + event user_silent + bot ask if more time needed +""" + + +def test_2(): + """Test a flow that uses a custom event, i.e., `user silent`.""" + config = RailsConfig.from_content(CONFIG_WITH_EVENT) + + chat = TestChat( + config, + llm_completions=[ + " express greeting", + ' "Hello!"', + ' "Do you need more time?"', + ], + ) + + events = [{"type": "user_said", "content": "Hello!"}] + new_events = chat.app.generate_events(events) + + assert {"type": "bot_said", "content": "Hello!"} in new_events + + events.extend(new_events) + events.append({"type": "user_silent"}) + + new_events = chat.app.generate_events(events) + + assert {"type": "bot_said", "content": "Do you need more time?"} in new_events + + +def test_3(): + """Test a flow that uses a custom event, i.e., `user silent` using the messages API.""" + config = RailsConfig.from_content(CONFIG_WITH_EVENT) + + chat = TestChat( + config, + llm_completions=[ + " express greeting", + ' "Hello!"', + ' "Do you need more time?"', + ], + ) + + messages = [{"role": "user", "content": "Hello!"}] + + new_message = chat.app.generate(messages=messages) + + assert new_message == {"role": "assistant", "content": "Hello!"} + + messages.append(new_message) + messages.append({"role": "event", "event": {"type": "user_silent"}}) + + new_message = chat.app.generate(messages=messages) + + assert new_message == {"role": "assistant", "content": "Do you need more time?"}