From 58ce34b826b5da00c5d3eabc67ea1e937624e97b Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 16:54:27 -0500 Subject: [PATCH 01/24] init: first package --- .../llama-index-packs-memary/.gitignore | 153 ++++++++++++++++++ .../llama-index-packs-memary/BUILD | 1 + .../llama-index-packs-memary/Makefile | 17 ++ .../llama-index-packs-memary/README.md | 1 + .../llama_index/packs/memary/__init__.py | 4 + .../llama-index-packs-memary/pyproject.toml | 56 +++++++ .../tests/__init__.py | 0 .../tests/test_packs_memary.py | 0 8 files changed, 232 insertions(+) create mode 100644 llama-index-packs/llama-index-packs-memary/.gitignore create mode 100644 llama-index-packs/llama-index-packs-memary/BUILD create mode 100644 llama-index-packs/llama-index-packs-memary/Makefile create mode 100644 llama-index-packs/llama-index-packs-memary/README.md create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py create mode 100644 llama-index-packs/llama-index-packs-memary/pyproject.toml create mode 100644 llama-index-packs/llama-index-packs-memary/tests/__init__.py create mode 100644 llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py diff --git a/llama-index-packs/llama-index-packs-memary/.gitignore b/llama-index-packs/llama-index-packs-memary/.gitignore new file mode 100644 index 0000000000000..990c18de22908 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/.gitignore @@ -0,0 +1,153 @@ +llama_index/_static +.DS_Store +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +bin/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +etc/ +include/ +lib/ +lib64/ +parts/ +sdist/ +share/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +.ruff_cache + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints +notebooks/ + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pyvenv.cfg + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Jetbrains +.idea +modules/ +*.swp + +# VsCode +.vscode + +# pipenv +Pipfile +Pipfile.lock + +# pyright +pyrightconfig.json diff --git a/llama-index-packs/llama-index-packs-memary/BUILD b/llama-index-packs/llama-index-packs-memary/BUILD new file mode 100644 index 0000000000000..db46e8d6c978c --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/llama-index-packs/llama-index-packs-memary/Makefile b/llama-index-packs/llama-index-packs-memary/Makefile new file mode 100644 index 0000000000000..b9eab05aa3706 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/Makefile @@ -0,0 +1,17 @@ +GIT_ROOT ?= $(shell git rev-parse --show-toplevel) + +help: ## Show all Makefile targets. + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}' + +format: ## Run code autoformatters (black). + pre-commit install + git ls-files | xargs pre-commit run black --files + +lint: ## Run linters: pre-commit (black, ruff, codespell) and mypy + pre-commit install && git ls-files | xargs pre-commit run --show-diff-on-failure --files + +test: ## Run tests via pytest. + pytest tests + +watch-docs: ## Build and watch documentation. + sphinx-autobuild docs/ docs/_build/html --open-browser --watch $(GIT_ROOT)/llama_index/ diff --git a/llama-index-packs/llama-index-packs-memary/README.md b/llama-index-packs/llama-index-packs-memary/README.md new file mode 100644 index 0000000000000..ecbafa911fc4c --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/README.md @@ -0,0 +1 @@ +# LlamaIndex Packs Integration: Memary diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py new file mode 100644 index 0000000000000..f4154a182435c --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py @@ -0,0 +1,4 @@ +from llama_index.packs.memary.base import + + +__all__ = [""] diff --git a/llama-index-packs/llama-index-packs-memary/pyproject.toml b/llama-index-packs/llama-index-packs-memary/pyproject.toml new file mode 100644 index 0000000000000..2e59c77f58dbf --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/pyproject.toml @@ -0,0 +1,56 @@ +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.codespell] +check-filenames = true +check-hidden = true +# Feel free to un-skip examples, and experimental, you will just need to +# work through many typos (--write-changes and --interactive will help) +skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb" + +# [tool.llamahub] +# contains_example = false +# import_path = "" + +# [tool.llamahub.class_authors] +# CLASS = "github-username" + +[tool.mypy] +disallow_untyped_defs = true +# Remove venv skip when integrated with pre-commit +exclude = ["_static", "build", "examples", "notebooks", "venv"] +ignore_missing_imports = true +python_version = "3.8" + +[tool.poetry] +name = "llama-index-packs-memary" +version = "0.1.0" +description = "llama-index packs memary integration" +authors = ["Your Name "] +license = "MIT" +readme = "README.md" +packages = [{include = "llama_index/"}] + +[tool.poetry.dependencies] +python = ">=3.8.1,<4.0" +llama-index-core = "^0.10.0" + +[tool.poetry.group.dev.dependencies] +black = {extras = ["jupyter"], version = "<=23.9.1,>=23.7.0"} +codespell = {extras = ["toml"], version = ">=v2.2.6"} +ipython = "8.10.0" +jupyter = "^1.0.0" +mypy = "0.991" +pre-commit = "3.2.0" +pylint = "2.15.10" +pytest = "7.2.1" +pytest-mock = "3.11.1" +ruff = "0.0.292" +tree-sitter-languages = "^1.8.0" +types-Deprecated = ">=0.1.0" +types-PyYAML = "^6.0.12.12" +types-protobuf = "^4.24.0.4" +types-redis = "4.5.5.0" +types-requests = "2.28.11.8" # TODO: unpin when mypy>0.991 +types-setuptools = "67.1.0.0" diff --git a/llama-index-packs/llama-index-packs-memary/tests/__init__.py b/llama-index-packs/llama-index-packs-memary/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py b/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py new file mode 100644 index 0000000000000..e69de29bb2d1d From 4c62740ea4a18d7ef9343ca0c56dcf1aa2eb2721 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:47:31 -0500 Subject: [PATCH 02/24] docs: add llamahub info --- .../llama-index-packs-memary/pyproject.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llama-index-packs/llama-index-packs-memary/pyproject.toml b/llama-index-packs/llama-index-packs-memary/pyproject.toml index 2e59c77f58dbf..7bddb0ab166dd 100644 --- a/llama-index-packs/llama-index-packs-memary/pyproject.toml +++ b/llama-index-packs/llama-index-packs-memary/pyproject.toml @@ -9,12 +9,12 @@ check-hidden = true # work through many typos (--write-changes and --interactive will help) skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb" -# [tool.llamahub] -# contains_example = false -# import_path = "" +[tool.llamahub] +contains_example = true +import_path = "llama_index.packs.memary" -# [tool.llamahub.class_authors] -# CLASS = "github-username" +[tool.llamahub.class_authors] +CLASS = "seyeong-han" [tool.mypy] disallow_untyped_defs = true From f6c6db16c5d8dcbe35cdcc13b6008b080acbee47 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:48:05 -0500 Subject: [PATCH 03/24] docs: versioning python and memary --- llama-index-packs/llama-index-packs-memary/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama-index-packs/llama-index-packs-memary/pyproject.toml b/llama-index-packs/llama-index-packs-memary/pyproject.toml index 7bddb0ab166dd..7fc879735a1c0 100644 --- a/llama-index-packs/llama-index-packs-memary/pyproject.toml +++ b/llama-index-packs/llama-index-packs-memary/pyproject.toml @@ -21,11 +21,11 @@ disallow_untyped_defs = true # Remove venv skip when integrated with pre-commit exclude = ["_static", "build", "examples", "notebooks", "venv"] ignore_missing_imports = true -python_version = "3.8" +python_version = "3.10.12" [tool.poetry] name = "llama-index-packs-memary" -version = "0.1.0" +version = "0.1.3" description = "llama-index packs memary integration" authors = ["Your Name "] license = "MIT" From d878bd07a5f519f2f81eee5fc250d42c46504a5f Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:48:19 -0500 Subject: [PATCH 04/24] docs: author's info --- llama-index-packs/llama-index-packs-memary/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-index-packs/llama-index-packs-memary/pyproject.toml b/llama-index-packs/llama-index-packs-memary/pyproject.toml index 7fc879735a1c0..6260321138c58 100644 --- a/llama-index-packs/llama-index-packs-memary/pyproject.toml +++ b/llama-index-packs/llama-index-packs-memary/pyproject.toml @@ -27,7 +27,7 @@ python_version = "3.10.12" name = "llama-index-packs-memary" version = "0.1.3" description = "llama-index packs memary integration" -authors = ["Your Name "] +authors = ["Seyeong Han "] license = "MIT" readme = "README.md" packages = [{include = "llama_index/"}] From 69dff3e0cc041b5cf45a073b846e438d79074fba Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:48:36 -0500 Subject: [PATCH 05/24] docs: add poetry dependencies --- .../llama-index-packs-memary/pyproject.toml | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/llama-index-packs/llama-index-packs-memary/pyproject.toml b/llama-index-packs/llama-index-packs-memary/pyproject.toml index 6260321138c58..58188acc07884 100644 --- a/llama-index-packs/llama-index-packs-memary/pyproject.toml +++ b/llama-index-packs/llama-index-packs-memary/pyproject.toml @@ -33,8 +33,34 @@ readme = "README.md" packages = [{include = "llama_index/"}] [tool.poetry.dependencies] -python = ">=3.8.1,<4.0" -llama-index-core = "^0.10.0" +python = ">3.9.7,<3.11.10" +neo4j="5.17.0" +python-dotenv="1.0.1" +pyvis="0.3.2" +streamlit="1.31.1" +llama-index="0.10.11" +llama-index-agent-openai="0.1.5" +llama-index-core="0.10.12" +llama-index-embeddings-openai="0.1.5" +llama-index-graph-stores-nebula="0.1.2" +llama-index-graph-stores-neo4j="0.1.1" +llama-index-legacy="0.9.48" +llama-index-llms-openai="0.1.5" +llama-index-multi-modal-llms-openai="0.1.3" +llama-index-program-openai="0.1.3" +llama-index-question-gen-openai="0.1.2" +llama-index-readers-file="0.1.4" +langchain="0.1.12" +langchain-openai="0.0.8" +llama-index-llms-perplexity="0.1.3" +llama-index-multi-modal-llms-ollama="0.1.3" +llama-index-llms-ollama="0.1.2" +pandas="2.2.0" +geocoder="1.38.1" +googlemaps="4.10.0" +ansistrip="0.1" +numpy="1.26.4" +ollama="0.1.9" [tool.poetry.group.dev.dependencies] black = {extras = ["jupyter"], version = "<=23.9.1,>=23.7.0"} From 054c71fa715646f80be83f7711fc656ccb45adec Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:48:57 -0500 Subject: [PATCH 06/24] init: first README --- .../llama-index-packs-memary/README.md | 112 +++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/llama-index-packs/llama-index-packs-memary/README.md b/llama-index-packs/llama-index-packs-memary/README.md index ecbafa911fc4c..3edd45ce77561 100644 --- a/llama-index-packs/llama-index-packs-memary/README.md +++ b/llama-index-packs/llama-index-packs-memary/README.md @@ -1 +1,111 @@ -# LlamaIndex Packs Integration: Memary +# memary Pack + +Agents use LLMs that are currently constrained to finite context windows. memary overcomes this limitation by allowing your agents to store a large corpus of information in knowledge graphs, infer user knowledge through our memory modules, and only retrieve relevant information for meaningful responses. + +## CLI Usage + +You can download llamapacks directly using `llamaindex-cli`, which comes installed with the `llama-index` python package: + +```bash +llamaindex-cli download-llamapack memary --download-dir ./memary +``` + +You can then inspect the files at `./memary` and use them as a template for your own project. + +## Demo + +**Notes:** memary currently assumes the local installation method and currently supports any models available through Ollama: + +- LLM running locally using Ollama (Llama 3 8B/40B as suggested defaults) **OR** `gpt-3.5-turbo` +- Vision model running locally using Ollama (LLaVA as suggested default) **OR** `gpt-4-vision-preview` + +memary will default to the locally run models unless explicitly specified. + +**To run the Streamlit app:** + +1. [Optional] If running models locally using Ollama, follow this the instructions in this [repo](https://github.com/ollama/ollama). + +2. Ensure that a `.env` exists with any necessary API keys and Neo4j credentials. + +``` +OPENAI_API_KEY="YOUR_API_KEY" +NEO4J_PW="YOUR_NEO4J_PW" +NEO4J_URL="YOUR_NEO4J_URL" +PERPLEXITY_API_KEY="YOUR_API_KEY" +GOOGLEMAPS_API_KEY="YOUR_API_KEY" +ALPHA_VANTAGE_API_KEY="YOUR_API_KEY" +``` + +3. How to get API keys: + +``` +OpenAI key: https://openai.com/index/openai-api + +Neo4j: https://neo4j.com/cloud/platform/aura-graph-database/?ref=nav-get-started-cta + Click 'Start for free' + Create a free instance + Open auto-downloaded txt file and use the credentials + +Perplexity key: https://www.perplexity.ai/settings/api + +Google Maps: + Keys are generated in the 'Credentials' page of the 'APIs & Services' tab of Google Cloud Console https://console.cloud.google.com/apis/credentials + +Alpha Vantage: (this key is for getting real time stock data) + https://www.alphavantage.co/support/#api-key + Reccomend use https://10minutemail.com/ to generate a temporary email to use +``` + +4. Update user persona which can be found in `streamlit_app/data/user_persona.txt` using the user persona template which can be found in `streamlit_app/data/user_persona_template.txt`. Instructions have been provided - replace the curly brackets with relevant information. + +5. . [Optional] Update system persona, if needed, which can be found in `streamlit_app/data/system_persona.txt`. +6. Run: + +``` +cd streamlit_app +streamlit run app.py +``` + +## Usage + +```python +from dotenv import load_dotenv +load_dotenv() + +from llama_index.packs.memary.agent.chat_agent import ChatAgent + +system_persona_txt = "data/system_persona.txt" +user_persona_txt = "data/user_persona.txt" +past_chat_json = "data/past_chat.json" +memory_stream_json = "data/memory_stream.json" +entity_knowledge_store_json = "data/entity_knowledge_store.json" + +chat_agent = ChatAgent( + "Personal Agent", + memory_stream_json, + entity_knowledge_store_json, + system_persona_txt, + user_persona_txt, + past_chat_json, +) +``` + +Pass in subset of `['search', 'vision', 'locate', 'stocks']` as `include_from_defaults` for different set of default tools upon initialization. + +### Adding Custom Tools + +```python +def multiply(a: int, b: int) -> int: + """Multiply two integers and returns the result integer""" + return a * b + +chat_agent.add_tool({"multiply": multiply}) +``` + +More information about creating custom tools for the LlamaIndex ReAct Agent can be found [here](https://docs.llamaindex.ai/en/stable/examples/agent/react_agent/). + +### Removing Tools + +```python +chat_agent.remove_tool("multiply") +``` From 4d710f9fa727b8a2aa0a7ae51fab4f4b941fc8eb Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:50:03 -0500 Subject: [PATCH 07/24] init: first memary/agent --- .../packs/memary/agent/__init__.py | 12 + .../packs/memary/agent/base_agent.py | 498 ++++++++++++++++++ .../packs/memary/agent/chat_agent.py | 87 +++ .../packs/memary/agent/data_types.py | 96 ++++ 4 files changed, 693 insertions(+) create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py new file mode 100644 index 0000000000000..9c98843a85b55 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py @@ -0,0 +1,12 @@ +from llama_index.packs.memary.agent.base_agent import Agent +from llama_index.packs.memary.agent.chat_agent import ChatAgent +from llama_index.packs.memary.agent.llm_api.tools import ( + ollama_chat_completions_request, openai_chat_completions_request) +from llama_index.packs.memary.memory import EntityKnowledgeStore, MemoryStream +from llama_index.packs.memary.synonym_expand.synonym import custom_synonym_expand_fn + +__all__ = [ + "Agent", "ChatAgent", "EntityKnowledgeStore", "MemoryStream", + "ollama_chat_completions_request", "openai_chat_completions_request", + "custom_synonym_expand_fn" +] diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py new file mode 100644 index 0000000000000..4ee68a715703c --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py @@ -0,0 +1,498 @@ +import logging +import os +import sys +from pathlib import Path +from typing import Any, Callable, Dict, List + +import geocoder +import googlemaps +import numpy as np +import requests +from ansistrip import ansi_strip +from dotenv import load_dotenv +from llama_index.core import (KnowledgeGraphIndex, Settings, + SimpleDirectoryReader, StorageContext) +from llama_index.core.agent import ReActAgent +from llama_index.core.llms import ChatMessage +from llama_index.core.query_engine import RetrieverQueryEngine +from llama_index.core.retrievers import KnowledgeGraphRAGRetriever +from llama_index.core.tools import FunctionTool +from llama_index.graph_stores.neo4j import Neo4jGraphStore +from llama_index.llms.ollama import Ollama +from llama_index.llms.openai import OpenAI +from llama_index.llms.perplexity import Perplexity +from llama_index.multi_modal_llms.ollama import OllamaMultiModal +from llama_index.multi_modal_llms.openai import OpenAIMultiModal + +from llama_index.packs.memary.agent.data_types import Context, Message +from llama_index.packs.memary.agent.llm_api.tools import ( + ollama_chat_completions_request, openai_chat_completions_request) +from llama_index.packs.memary.memory import EntityKnowledgeStore, MemoryStream +from llama_index.packs.memary.synonym_expand import custom_synonym_expand_fn + +MAX_ENTITIES_FROM_KG = 5 +ENTITY_EXCEPTIONS = ["Unknown relation"] +# LLM token limits +CONTEXT_LENGTH = 4096 +EVICTION_RATE = 0.7 +NONEVICTION_LENGTH = 5 +TOP_ENTITIES = 20 + + +def generate_string(entities): + cypher_query = "MATCH p = (n) - [*1 .. 2] - ()\n" + cypher_query += "WHERE n.id IN " + str(entities) + "\n" + cypher_query += "RETURN p" + + return cypher_query + + +class Agent(object): + """Agent manages the RAG model, the ReAct agent, and the memory stream.""" + + def __init__( + self, + name, + memory_stream_json, + entity_knowledge_store_json, + system_persona_txt, + user_persona_txt, + past_chat_json, + llm_model_name="llama3", + vision_model_name="llava", + include_from_defaults=["search", "locate", "vision", "stocks"], + debug=True, + ): + load_dotenv() + self.name = name + self.model = llm_model_name + + googlemaps_api_key = os.getenv("GOOGLEMAPS_API_KEY") + pplx_api_key = os.getenv("PERPLEXITY_API_KEY") + + # Neo4j credentials + self.neo4j_username = "neo4j" + self.neo4j_password = os.getenv("NEO4J_PW") + self.neo4j_url = os.getenv("NEO4J_URL") + database = "neo4j" + + # initialize APIs + self.load_llm_model(llm_model_name) + self.load_vision_model(vision_model_name) + self.query_llm = Perplexity(api_key=pplx_api_key, + model="mistral-7b-instruct", + temperature=0.5) + self.gmaps = googlemaps.Client(key=googlemaps_api_key) + Settings.llm = self.llm + Settings.chunk_size = 512 + + # initialize Neo4j graph resources + self.graph_store = Neo4jGraphStore( + username=self.neo4j_username, + password=self.neo4j_password, + url=self.neo4j_url, + database=database, + ) + + self.vantage_key = os.getenv("ALPHA_VANTAGE_API_KEY") + + self.storage_context = StorageContext.from_defaults( + graph_store=self.graph_store) + graph_rag_retriever = KnowledgeGraphRAGRetriever( + storage_context=self.storage_context, + verbose=True, + llm=self.llm, + retriever_mode="keyword", + synonym_expand_fn=custom_synonym_expand_fn, + ) + + self.query_engine = RetrieverQueryEngine.from_args( + graph_rag_retriever, ) + + self.debug = debug + self.tools = {} + self._init_default_tools(default_tools=include_from_defaults) + + self.memory_stream = MemoryStream(memory_stream_json) + self.entity_knowledge_store = EntityKnowledgeStore( + entity_knowledge_store_json) + + self.message = Message(system_persona_txt, user_persona_txt, + past_chat_json, self.model) + + def __str__(self): + return f"Agent {self.name}" + + def load_llm_model(self, llm_model_name): + if llm_model_name == "gpt-3.5-turbo": + os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY") + self.openai_api_key = os.environ["OPENAI_API_KEY"] + self.model_endpoint = "https://api.openai.com/v1" + self.llm = OpenAI(model="gpt-3.5-turbo-instruct") + else: + try: + self.llm = Ollama(model=llm_model_name, request_timeout=60.0) + except: + raise ("Please provide a proper llm_model_name.") + + def load_vision_model(self, vision_model_name): + if vision_model_name == "gpt-4-vision-preview": + os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY") + self.openai_api_key = os.environ["OPENAI_API_KEY"] + self.mm_model = OpenAIMultiModal( + model="gpt-4-vision-preview", + api_key=os.getenv("OPENAI_KEY"), + max_new_tokens=300, + ) + else: + try: + self.mm_model = OllamaMultiModal(model=vision_model_name) + except: + raise ("Please provide a proper vision_model_name.") + + def external_query(self, query: str): + messages_dict = [ + { + "role": "system", + "content": "Be precise and concise." + }, + { + "role": "user", + "content": query + }, + ] + messages = [ChatMessage(**msg) for msg in messages_dict] + external_response = self.query_llm.chat(messages) + + return str(external_response) + + def search(self, query: str) -> str: + """Search the knowledge graph or perform search on the web if information is not present in the knowledge graph""" + response = self.query_engine.query(query) + + if response.metadata is None: + return self.external_query(query) + else: + return response + + def locate(self, query: str) -> str: + """Finds the current geographical location""" + location = geocoder.ip("me") + lattitude, longitude = location.latlng[0], location.latlng[1] + + reverse_geocode_result = self.gmaps.reverse_geocode( + (lattitude, longitude)) + formatted_address = reverse_geocode_result[0]["formatted_address"] + return "Your address is" + formatted_address + + def vision(self, query: str, img_url: str) -> str: + """Uses computer vision to process the image specified by the image url and answers the question based on the CV results""" + query_image_dir_path = Path("query_images") + if not query_image_dir_path.exists(): + Path.mkdir(query_image_dir_path) + + data = requests.get(img_url).content + query_image_path = os.path.join(query_image_dir_path, "query.jpg") + with open(query_image_path, "wb") as f: + f.write(data) + image_documents = SimpleDirectoryReader( + query_image_dir_path).load_data() + + response = self.mm_model.complete(prompt=query, + image_documents=image_documents) + + os.remove(query_image_path) # delete image after use + return response + + def stocks(self, query: str) -> str: + """Get the stock price of the company given the ticker""" + request_api = requests.get( + r"https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol=" + + query + r"&apikey=" + self.vantage_key) + return request_api.json() + + # def get_news(self, query: str) -> str: + # """Given a keyword, search for news articles related to the keyword""" + # request_api = requests.get(r'https://newsdata.io/api/1/news?apikey=' + self.news_data_key + r'&q=' + query) + # return request_api.json() + + def query(self, query: str) -> str: + # get the response from react agent + response = self.routing_agent.chat(query) + self.routing_agent.reset() + # write response to file for KG writeback + with open("data/external_response.txt", "w") as f: + print(response, file=f) + # write back to the KG + self.write_back() + return response + + def write_back(self): + documents = SimpleDirectoryReader( + input_files=["data/external_response.txt"]).load_data() + + KnowledgeGraphIndex.from_documents( + documents, + storage_context=self.storage_context, + max_triplets_per_chunk=8, + ) + + def check_KG(self, query: str) -> bool: + """Check if the query is in the knowledge graph. + + Args: + query (str): query to check in the knowledge graph + + Returns: + bool: True if the query is in the knowledge graph, False otherwise + """ + response = self.query_engine.query(query) + + if response.metadata is None: + return False + return generate_string( + list(list(response.metadata.values())[0]["kg_rel_map"].keys())) + + def _select_top_entities(self): + entity_knowledge_store = self.message.llm_message[ + "knowledge_entity_store"] + entities = [entity.to_dict() for entity in entity_knowledge_store] + entity_counts = [entity["count"] for entity in entities] + top_indexes = np.argsort(entity_counts)[:TOP_ENTITIES] + return [entities[index] for index in top_indexes] + + def _add_contexts_to_llm_message(self, role, content, index=None): + """Add contexts to the llm_message.""" + if index: + self.message.llm_message["messages"].insert( + index, Context(role, content)) + else: + self.message.llm_message["messages"].append(Context(role, content)) + + def _change_llm_message_chat(self) -> dict: + """Change the llm_message to chat format. + + Returns: + dict: llm_message in chat format + """ + llm_message_chat = self.message.llm_message.copy() + llm_message_chat["messages"] = [] + top_entities = self._select_top_entities() + logging.info(f"top_entities: {top_entities}") + llm_message_chat["messages"].append({ + "role": + "user", + "content": + "Knowledge Entity Store:" + str(top_entities), + }) + llm_message_chat["messages"].extend([ + context.to_dict() + for context in self.message.llm_message["messages"] + ]) + llm_message_chat.pop("knowledge_entity_store") + llm_message_chat.pop("memory_stream") + return llm_message_chat + + def _summarize_contexts(self, total_tokens: int): + """Summarize the contexts. + + Args: + total_tokens (int): total tokens in the response + """ + messages = self.message.llm_message["messages"] + + # First two messages are system and user personas + if len(messages) > 2 + NONEVICTION_LENGTH: + messages = messages[2:-NONEVICTION_LENGTH] + del self.message.llm_message["messages"][2:-NONEVICTION_LENGTH] + else: + messages = messages[2:] + del self.message.llm_message["messages"][2:] + + message_contents = [ + message.to_dict()["content"] for message in messages + ] + + llm_message_chat = { + "model": + self.model, + "messages": [{ + "role": + "user", + "content": + "Summarize these previous conversations into 50 words:" + + str(message_contents), + }], + } + response, _ = self._get_chat_response(llm_message_chat) + content = "Summarized past conversation:" + response + self._add_contexts_to_llm_message("assistant", content, index=2) + logging.info( + f"Contexts summarized successfully. \n summary: {response}") + logging.info( + f"Total tokens after eviction: {total_tokens*EVICTION_RATE}") + + def _get_chat_response(self, llm_message_chat: str) -> str: + """Get response from the LLM chat model. + + Args: + llm_message_chat (str): query to get response for + + Returns: + str: response from the LLM chat model + """ + if self.model == "gpt-3.5-turbo": + response = openai_chat_completions_request(self.model_endpoint, + self.openai_api_key, + llm_message_chat) + total_tokens = response["usage"]["total_tokens"] + response = str(response["choices"][0]["message"]["content"]) + else: # default to Ollama model + response = ollama_chat_completions_request( + llm_message_chat["messages"], self.model) + total_tokens = response.get( + "prompt_eval_count", + 0) # if 'prompt_eval_count' not present then query is cached + response = str(response["message"]["content"]) + return response, total_tokens + + def get_response(self) -> str: + """Get response from the RAG model. + + Returns: + str: response from the RAG model + """ + llm_message_chat = self._change_llm_message_chat() + response, total_tokens = self._get_chat_response(llm_message_chat) + if total_tokens > CONTEXT_LENGTH * EVICTION_RATE: + logging.info("Evicting and summarizing contexts") + self._summarize_contexts(total_tokens) + + self.message.save_contexts_to_json() + + return response + + def get_routing_agent_response(self, query, return_entity=False): + """Get response from the ReAct.""" + response = "" + if self.debug: + # writes ReAct agent steps to separate file and modifies format to be readable in .txt file + with open("data/routing_response.txt", "w") as f: + orig_stdout = sys.stdout + sys.stdout = f + response = str(self.query(query)) + sys.stdout.flush() + sys.stdout = orig_stdout + text = "" + with open("data/routing_response.txt", "r") as f: + text = f.read() + + plain = ansi_strip(text) + with open("data/routing_response.txt", "w") as f: + f.write(plain) + else: + response = str(self.query(query)) + + if return_entity: + # the query above already adds final response to KG so entities will be present in the KG + return response, self.get_entity(self.query_engine.retrieve(query)) + return response + + def get_entity(self, retrieve) -> list[str]: + """retrieve is a list of QueryBundle objects. + A retrieved QueryBundle object has a "node" attribute, + which has a "metadata" attribute. + + example for "kg_rel_map": + kg_rel_map = { + 'Harry': [['DREAMED_OF', 'Unknown relation'], ['FELL_HARD_ON', 'Concrete floor']], + 'Potter': [['WORE', 'Round glasses'], ['HAD', 'Dream']] + } + + Args: + retrieve (list[NodeWithScore]): list of NodeWithScore objects + return: + list[str]: list of string entities + """ + + entities = [] + kg_rel_map = retrieve[0].node.metadata["kg_rel_map"] + for key, items in kg_rel_map.items(): + # key is the entity of question + entities.append(key) + # items is a list of [relationship, entity] + entities.extend(item[1] for item in items) + if len(entities) > MAX_ENTITIES_FROM_KG: + break + entities = list(set(entities)) + for exceptions in ENTITY_EXCEPTIONS: + if exceptions in entities: + entities.remove(exceptions) + return entities + + def _init_ReAct_agent(self): + """Initializes ReAct Agent with list of tools in self.tools.""" + tool_fns = [] + for func in self.tools.values(): + tool_fns.append(FunctionTool.from_defaults(fn=func)) + self.routing_agent = ReActAgent.from_tools(tool_fns, + llm=self.llm, + verbose=True) + + def _init_default_tools(self, default_tools: List[str]): + """Initializes ReAct Agent from the default list of tools memary provides. + List of strings passed in during initialization denoting which default tools to include. + Args: + default_tools (list(str)): list of tool names in string form + """ + + for tool in default_tools: + if tool == "search": + self.tools["search"] = self.search + elif tool == "locate": + self.tools["locate"] = self.locate + elif tool == "vision": + self.tools["vision"] = self.vision + elif tool == "stocks": + self.tools["stocks"] = self.stocks + self._init_ReAct_agent() + + def add_tool(self, tool_additions: Dict[str, Callable[..., Any]]): + """Adds specified tools to be used by the ReAct Agent. + Args: + tools (dict(str, func)): dictionary of tools with names as keys and associated functions as values + """ + + for tool_name in tool_additions: + self.tools[tool_name] = tool_additions[tool_name] + self._init_ReAct_agent() + + def remove_tool(self, tool_name: str): + """Removes specified tool from list of available tools for use by the ReAct Agent. + Args: + tool_name (str): name of tool to be removed in string form + """ + + if tool_name in self.tools: + del self.tools[tool_name] + self._init_ReAct_agent() + else: + raise ("Unknown tool_name provided for removal.") + + def update_tools(self, updated_tools: List[str]): + """Resets ReAct Agent tools to only include subset of default tools. + Args: + updated_tools (list(str)): list of default tools to include + """ + + self.tools.clear() + for tool in updated_tools: + if tool == "search": + self.tools["search"] = self.search + elif tool == "locate": + self.tools["locate"] = self.locate + elif tool == "vision": + self.tools["vision"] = self.vision + elif tool == "stocks": + self.tools["stocks"] = self.stocks + self._init_ReAct_agent() diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py new file mode 100644 index 0000000000000..ce4c25513b0c9 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py @@ -0,0 +1,87 @@ +from typing import Optional, List + +from llama_index.packs.memary.agent.base_agent import Agent +import logging + + +class ChatAgent(Agent): + """ChatAgent currently able to support Llama3 running on Ollama (default) and gpt-3.5-turbo for llm models, + and LLaVA running on Ollama (default) and gpt-4-vision-preview for the vision tool. + """ + + def __init__( + self, + name, + memory_stream_json, + entity_knowledge_store_json, + system_persona_txt, + user_persona_txt, + past_chat_json, + llm_model_name="llama3", + vision_model_name="llava", + include_from_defaults=["search", "locate", "vision", "stocks"], + ): + super().__init__( + name, + memory_stream_json, + entity_knowledge_store_json, + system_persona_txt, + user_persona_txt, + past_chat_json, + llm_model_name, + vision_model_name, + include_from_defaults, + ) + + def add_chat(self, + role: str, + content: str, + entities: Optional[List[str]] = None): + """Add a chat to the agent's memory. + + Args: + role (str): 'system' or 'user' + content (str): content of the chat + entities (Optional[List[str]], optional): entities from Memory systems. Defaults to None. + """ + # Add a chat to the agent's memory. + self._add_contexts_to_llm_message(role, content) + + if entities: + self.memory_stream.add_memory(entities) + self.memory_stream.save_memory() + self.entity_knowledge_store.add_memory( + self.memory_stream.get_memory()) + self.entity_knowledge_store.save_memory() + + self._replace_memory_from_llm_message() + self._replace_eks_to_from_message() + + def get_chat(self): + return self.contexts + + def clearMemory(self): + """Clears Neo4j database and memory stream/entity knowledge store.""" + + logging.info("Deleting memory stream and entity knowledge store...") + self.memory_stream.clear_memory() + self.entity_knowledge_store.clear_memory() + + logging.info("Deleting nodes from Neo4j...") + try: + self.graph_store.query("MATCH (n) DETACH DELETE n") + except Exception as e: + logging.error(f"Error deleting nodes: {e}") + logging.info("Nodes deleted from Neo4j.") + + def _replace_memory_from_llm_message(self): + """Replace the memory_stream from the llm_message.""" + self.message.llm_message[ + "memory_stream"] = self.memory_stream.get_memory() + + def _replace_eks_to_from_message(self): + """Replace the entity knowledge store from the llm_message. + eks = entity knowledge store""" + + self.message.llm_message["knowledge_entity_store"] = ( + self.entity_knowledge_store.get_memory()) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py new file mode 100644 index 0000000000000..55ce685ace785 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py @@ -0,0 +1,96 @@ +import json +import logging +from dataclasses import dataclass + + +def save_json(filename, data): + with open(filename, "w") as f: + json.dump(data, f, indent=4) + + +@dataclass +class Context: + """Context class to store the role and content of the message.""" + role: str # system or user + content: str + + def __str__(self): + return f"{self.role}: {self.content} |" + + def to_dict(self): + return {'role': self.role, 'content': self.content} + + +class Message: + """Message class to store the contexts, memory stream and knowledge entity store.""" + + def __init__(self, system_persona_txt, user_persona_txt, past_chat_json, + model): + self.past_chat_json = past_chat_json + + self.contexts = [] + self.system_persona = self.load_persona(system_persona_txt) + self.user_persona = self.load_persona(user_persona_txt) + self._init_persona_to_messages() + self.contexts.extend(self.load_contexts_from_json()) + + self.llm_message = { + "model": model, + "messages": self.contexts, + "memory_stream": [], + "knowledge_entity_store": [] + } + + # self.prompt_tokens = count_tokens(self.contexts) + + def __str__(self): + llm_message_str = f"System Persona: {self.system_persona}\nUser Persona: {self.user_persona}\n" + for context in self.contexts: + llm_message_str += f"{str(context)}," + for memory in self.llm_message["memory_stream"]: + llm_message_str += f"{str(memory)}," + for entity in self.llm_message["knowledge_entity_store"]: + llm_message_str += f"{str(entity)}," + return llm_message_str + + def _init_persona_to_messages(self): + """Initializes the system and user personas to the contexts.""" + self.contexts.append(Context("system", self.system_persona)) + self.contexts.append(Context("user", self.user_persona)) + + def load_persona(self, persona_txt) -> str: + """Loads the persona from the txt file. + + Args: + persona_txt (str): persona txt file path + + Returns: + str: persona + """ + try: + with open(persona_txt, "r") as file: + persona = file.read() + return persona + except FileNotFoundError: + logging.info(f"{persona_txt} file does not exist.") + + def load_contexts_from_json(self): + """Loads the contexts from the past chat json file.""" + try: + with open(self.past_chat_json, "r") as file: + data_dicts = json.load(file) + + return [Context(**data_dict) for data_dict in data_dicts] + except: + logging.info( + f"{self.past_chat_json} file does not exist. Starts from empty contexts." + ) + return [] + + def save_contexts_to_json(self): + """Saves the contexts to the json file. + We don't save the system and user personas (first two messages) + """ + save_json(self.past_chat_json, [ + message.to_dict() for message in self.llm_message['messages'][2:] + ]) From ceef36736c6f4338cf2a343a6ad12fb6fc708f2d Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:50:29 -0500 Subject: [PATCH 08/24] init: first memary/memory --- .../packs/memary/memory/__init__.py | 5 ++ .../packs/memary/memory/base_memory.py | 89 +++++++++++++++++++ .../packs/memary/memory/data_types.py | 42 +++++++++ .../memary/memory/entity_knowledge_store.py | 87 ++++++++++++++++++ .../packs/memary/memory/memory_stream.py | 47 ++++++++++ 5 files changed, 270 insertions(+) create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/__init__.py create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/__init__.py new file mode 100644 index 0000000000000..9b9747d074f94 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/__init__.py @@ -0,0 +1,5 @@ +from llama_index.packs.memary.memory.base_memory import BaseMemory +from llama_index.packs.memary.memory.memory_stream import MemoryStream +from llama_index.packs.memary.memory.entity_knowledge_store import EntityKnowledgeStore + +__all__ = ["BaseMemory", "MemoryStream", "EntityKnowledgeStore"] diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py new file mode 100644 index 0000000000000..fef2da553dee3 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py @@ -0,0 +1,89 @@ +import json +import logging + +from abc import ABC, abstractmethod +from datetime import datetime, timedelta + + +class BaseMemory(ABC): + + def __init__(self, file_name: str, entity: str = None): + """Initializes the memory storage.""" + self.file_name = file_name + self.entity = entity + self.memory = [] + self.knowledge_memory = [] + self.init_memory() + + @abstractmethod + def __len__(self): + """Returns the number of items in the memory.""" + pass + + @abstractmethod + def init_memory(self): + """Initializes memory.""" + pass + + @abstractmethod + def load_memory_from_file(self): + """Loads memory from a file.""" + pass + + @abstractmethod + def add_memory(self, data): + """Adds new memory data.""" + pass + + @abstractmethod + def get_memory(self): + pass + + @property + def return_memory(self): + return self.memory + + def remove_old_memory(self, days): + """Removes memory items older than a specified number of days.""" + cutoff_date = datetime.now() - timedelta(days=days) + self.memory = [ + item for item in self.return_memory if item.date >= cutoff_date + ] + logging.info("Old memory removed successfully.") + + def save_memory(self): + if self.file_name: + with open(self.file_name, 'w') as file: + json.dump([item.to_dict() for item in self.return_memory], + file, + default=str, + indent=4) + logging.info(f"Memory saved to {self.file_name} successfully.") + else: + logging.info("No file name provided. Memory not saved.") + + def get_memory_by_index(self, index): + if 0 <= index < len(self.memory): + return self.memory[index] + else: + return None + + def remove_memory_by_index(self, index): + if 0 <= index < len(self.memory): + del self.memory[index] + logging.info("Memory item removed successfully.") + return True + else: + logging.info("Invalid index. Memory item not removed.") + return False + + def clear_memory(self): + self.memory = [] + if self.file_name: + with open(self.file_name, 'w') as file: + json.dump([], file, indent=4) + logging.info( + f"Memory cleared and saved to {self.file_name} successfully." + ) + else: + logging.info("No file name provided. Memory not cleared or saved.") diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py new file mode 100644 index 0000000000000..691990ea34266 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass, asdict +from datetime import datetime + + +@dataclass +class MemoryItem: + entity: str + date: datetime + + def __str__(self): + return f"{self.entity}, {self.date.isoformat()}" + + def to_dict(self): + return {'entity': self.entity, 'date': str(self.date.isoformat())} + + @classmethod + def from_dict(cls, data): + return cls(entity=data['entity'], + date=datetime.fromisoformat(data['date'])) + + +@dataclass +class KnowledgeMemoryItem: + entity: str + count: int + date: datetime + + def __str__(self): + return f"{self.entity}, {self.count}, {self.date.isoformat()}" + + def to_dict(self): + return { + 'entity': self.entity, + 'count': self.count, + 'date': str(self.date.isoformat()) + } + + @classmethod + def from_dict(cls, data): + return cls(entity=data['entity'], + count=data['count'], + date=datetime.fromisoformat(data['date'])) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py new file mode 100644 index 0000000000000..43ad1af78a1b3 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py @@ -0,0 +1,87 @@ +import json +import logging + +from llama_index.packs.memary.memory import BaseMemory +from llama_index.packs.memary.memory.data_types import KnowledgeMemoryItem, MemoryItem + + +class EntityKnowledgeStore(BaseMemory): + + def __len__(self): + """Returns the number of items in the memory.""" + return len(self.knowledge_memory) + + def init_memory(self): + """Initializes memory. + self.entity_memory: list[EntityMemoryItem] + """ + self.load_memory_from_file() + if self.entity: + self.add_memory(self.entity) + + @property + def return_memory(self): + return self.knowledge_memory + + def load_memory_from_file(self): + try: + with open(self.file_name, 'r') as file: + self.knowledge_memory = [ + KnowledgeMemoryItem.from_dict(item) + for item in json.load(file) + ] + logging.info( + f"Entity Knowledge Memory loaded from {self.file_name} successfully." + ) + except FileNotFoundError: + logging.info( + "File not found. Starting with an empty entity knowledge memory." + ) + + def add_memory(self, memory_stream: list[MemoryItem]): + """To add new memory to the entity knowledge store + we should convert the memory to knowledge memory and then update the knowledge memory + + Args: + memory_stream (list): list of MemoryItem + """ + knowledge_meory = self._convert_memory_to_knowledge_memory( + memory_stream) + self._update_knowledge_memory(knowledge_meory) + + def _update_knowledge_memory(self, knowledge_memory: list): + """update self.knowledge memory with new knowledge memory items + + Args: + knowledge_memory (list): list of KnowledgeMemoryItem + """ + for item in knowledge_memory: + for i, entity in enumerate(self.knowledge_memory): + if entity.entity == item.entity: + self.knowledge_memory[i].date = item.date + self.knowledge_memory[i].count += item.count + break + else: + self.knowledge_memory.append(item) + + def _convert_memory_to_knowledge_memory( + self, memory_stream: list) -> list[KnowledgeMemoryItem]: + """Converts memory to knowledge memory + + Returns: + knowledge_memory (list): list of KnowledgeMemoryItem + """ + knowledge_memory = [] + + entities = set([item.entity for item in memory_stream]) + for entity in entities: + memory_dates = [ + item.date for item in memory_stream if item.entity == entity + ] + knowledge_memory.append( + KnowledgeMemoryItem(entity, len(memory_dates), + max(memory_dates))) + return knowledge_memory + + def get_memory(self) -> list[KnowledgeMemoryItem]: + return self.knowledge_memory diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py new file mode 100644 index 0000000000000..f925ad3d4227c --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py @@ -0,0 +1,47 @@ +import json +import logging +from datetime import datetime + +from llama_index.packs.memary.memory import BaseMemory +from llama_index.packs.memary.memory.data_types import MemoryItem + +logging.basicConfig(level=logging.INFO) + + +class MemoryStream(BaseMemory): + + def __len__(self): + """Returns the number of items in the memory.""" + return len(self.memory) + + def init_memory(self): + """Initializes memory + self.memory: list[MemoryItem] + """ + self.load_memory_from_file() + if self.entity: + self.add_memory(self.entity) + + @property + def return_memory(self): + return self.memory + + def add_memory(self, entities): + self.memory.extend([ + MemoryItem(str(entity), + datetime.now().replace(microsecond=0)) + for entity in entities + ]) + + def get_memory(self) -> list[MemoryItem]: + return self.memory + + def load_memory_from_file(self): + try: + with open(self.file_name, 'r') as file: + self.memory = [ + MemoryItem.from_dict(item) for item in json.load(file) + ] + logging.info(f"Memory loaded from {self.file_name} successfully.") + except FileNotFoundError: + logging.info("File not found. Starting with an empty memory.") From 7f6e32cd01de8c1cda9879639b91198a948d31ff Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:50:54 -0500 Subject: [PATCH 09/24] init: first memary/synonym_expand --- .../packs/memary/synonym_expand/__init__.py | 4 ++ .../packs/memary/synonym_expand/output.py | 10 ++++ .../packs/memary/synonym_expand/synonym.py | 49 +++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py new file mode 100644 index 0000000000000..9c3dd1e6ff88e --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py @@ -0,0 +1,4 @@ +from llama_index.packs.memary.synonym_expand.synonym import custom_synonym_expand_fn +from llama_index.packs.memary.synonym_expand.output import SynonymOutput + +__all__ = ["custom_synonym_expand_fn", "SynonymOutput"] \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py new file mode 100644 index 0000000000000..56818a1198c00 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py @@ -0,0 +1,10 @@ +from typing import List + +from langchain_core.pydantic_v1 import BaseModel, Field + + +class SynonymOutput(BaseModel): + synonyms: List[str] = Field( + description= + "Synonyms of keywords provided, make every letter lowercase except for the first letter" + ) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py new file mode 100644 index 0000000000000..2e7c19f257647 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py @@ -0,0 +1,49 @@ +from langchain_openai import OpenAI +from langchain.prompts import PromptTemplate +from langchain_core.output_parsers import JsonOutputParser +from typing import List +import os +from llama_index.packs.memary.synonym_expand.output import SynonymOutput +from dotenv import load_dotenv + + +def custom_synonym_expand_fn(keywords: str) -> List[str]: + load_dotenv() + llm = OpenAI(openai_api_key=os.getenv("OPENAI_KEY"), temperature=0) + parser = JsonOutputParser(pydantic_object=SynonymOutput) + + template = """ + You are an expert synonym exapnding system. Find synonyms or words commonly used in place to reference the same word for every word in the list: + + Some examples are: + - a synonym for Palantir may be Palantir technologies or Palantir technologies inc. + - a synonym for Austin may be Austin texas + - a synonym for Taylor swift may be Taylor + - a synonym for Winter park may be Winter park resort + + Format: {format_instructions} + + Text: {keywords} + """ + + prompt = PromptTemplate( + template=template, + input_variables=["keywords"], + partial_variables={ + "format_instructions": parser.get_format_instructions() + }, + ) + + chain = prompt | llm | parser + result = chain.invoke({"keywords": keywords}) + + l = [] + for category in result: + for synonym in result[category]: + l.append(synonym.capitalize()) + + return l + + +# testing +# print(custom_synonym_expand_fn("[Nvidia]")) From b25fdf3e9a98aa2a52fa70022415f94c95a85815 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:52:17 -0500 Subject: [PATCH 10/24] init: memory module tests for pytest --- .../tests/data/external_response.txt | 0 .../tests/data/memory_stream.json | 1 + .../tests/data/past_chat.json | 1 + .../tests/data/routing_response.txt | 0 .../tests/data/system_persona.txt | 11 +++ .../tests/data/user_persona.txt | 9 +++ .../tests/data/user_persona_template.txt | 9 +++ .../tests/test_packs_memary.py | 75 +++++++++++++++++++ 8 files changed, 106 insertions(+) create mode 100644 llama-index-packs/llama-index-packs-memary/tests/data/external_response.txt create mode 100644 llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json create mode 100644 llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json create mode 100644 llama-index-packs/llama-index-packs-memary/tests/data/routing_response.txt create mode 100644 llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt create mode 100644 llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt create mode 100644 llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/external_response.txt b/llama-index-packs/llama-index-packs-memary/tests/data/external_response.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json b/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json new file mode 100644 index 0000000000000..0637a088a01e8 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json b/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json new file mode 100644 index 0000000000000..0637a088a01e8 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/routing_response.txt b/llama-index-packs/llama-index-packs-memary/tests/data/routing_response.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt b/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt new file mode 100644 index 0000000000000..bb28692261a77 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt @@ -0,0 +1,11 @@ +You are tasked with acting as a sophisticated conversational agent, uniquely equipped with a dual-layered memory system. This system consists of a Memory Stream and an Entity Knowledge Store. The Memory Stream captures all entities involved in conversations which are stored in knowledge graphs to infer users’ breadth of knowledge - ranging from questions and answers to their timestamps. Simultaneously, the Entity Knowledge Store captures the frequency and recency of the entities stored in the Memory Stream to infer users’ depth of knowledge. +Your primary role is to deliver a personalized and context-relevant final response that is two sentences or less by using the most recently added user query and ReAct agent response as primary information, and combining that with other information accessible via recent interactions and the top entities stored in the Entity Knowledge Store. You must comprehend the user's persona, incorporating their experiences, preferences, and personal details into your knowledge base outputting the final response. +The final response should not include ReAct agent information outside of the most recently added one unless relevant to the most recently asked question. +You are to interpret and apply the following data structures for personalized responses. Use the latest ReAct agent entry, Entity Knowledge Store, and Contexts to provide a final response to the latest question: + +User Persona: Information about the user's experience, preferences, and personal details. +ReAct agent: Answer generated by the routing agent using its tools. Represents the agent responses to different queries. The most recent ReAct agent response is the focus of the current final response. +Entity Knowledge Store: A record of entities, including a count reflecting their frequency and the date of the last time the entity was inserted into the Memory Stream. If any of these entities, especially those with high counts, are included in the agent response, don't elaborate or explain the concept - you can infer the user is well familiar with the concepts. Do not repeat the Entity Knowledge Store back in the response. +Interaction Keys: 'user' for user questions and 'system' for system-generated answers + +Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user’s questions and the overarching conversation context. When addressing the user’s latest inquiry, your answer must integrate the current conversation's context, historical interactions, the latest response from the ReAct agent as well as the top chosen entities from the entity knowledge store. Keep final responses to two sentences. \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt new file mode 100644 index 0000000000000..d4c9b144e07e5 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt @@ -0,0 +1,9 @@ +[Personal Information] +My name is Seyeong Han from South Korea. +I identify as a male and 28-year-old master's student at the University of Texas at Austin. +I have worked as an ML Engineer for three years in the Computer Vision area. +I really love sports such as badminton, tennis and workout and spend 4 or 5 days in exercise. + +[Answer Instructions] +I hope you can remember my questions and their answers so that I can leverage my personal past knowledge from you to be helpful in my life. +I always prefer short and concise answers, not over two sentences. \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt new file mode 100644 index 0000000000000..1c72b94203486 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt @@ -0,0 +1,9 @@ +[Personal Information] +My name is [name] from [country]. +I identify as a [insert age, sex, profession and any other relevant information to have the system know you best] +I have worked as [insert experience, field involved in, length of experience and any other relevant information to describe your capabilities] +[insert any other personal hobbies outside of work] + +[Answer Instructions] +Remember my past questions and their answers so that you can leverage my past knowledge and experiences to assist me in future interactions. +I always prefer short and concise answers, not over two sentences. \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py b/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py index e69de29bb2d1d..9f7b3f09a2991 100644 --- a/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py +++ b/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py @@ -0,0 +1,75 @@ +import os +import unittest +from datetime import datetime, timedelta +from llama_index.packs.memary import ChatAgent, EntityKnowledgeStore, MemoryStream +from llama_index.packs.memary.memory.data_types import KnowledgeMemoryItem, MemoryItem + + +class TestEntityKnowledgeStore(unittest.TestCase): + + def setUp(self): + self.file_name = "tests/memory/test_knowledge_memory.json" + self.entity_knowledge_store = EntityKnowledgeStore( + file_name=self.file_name) + + def tearDown(self): + # Clean up test file after each test + try: + os.remove(self.file_name) + except FileNotFoundError: + pass + + def test_add_memory(self): + data = [ + MemoryItem("test_entity", + datetime.now().replace(microsecond=0)) + ] + self.entity_knowledge_store.add_memory(data) + assert len(self.entity_knowledge_store.knowledge_memory) == 1 + assert isinstance(self.entity_knowledge_store.knowledge_memory[0], + KnowledgeMemoryItem) + + def test_convert_memory_to_knowledge_memory(self): + data = [ + MemoryItem("test_entity", + datetime.now().replace(microsecond=0)) + ] + converted_data = self.entity_knowledge_store._convert_memory_to_knowledge_memory( + data) + assert len(converted_data) == 1 + assert isinstance(converted_data[0], KnowledgeMemoryItem) + + def test_update_knowledge_memory(self): + data = [ + KnowledgeMemoryItem("knowledge_entity", 1, + datetime.now().replace(microsecond=0)) + ] + self.entity_knowledge_store._update_knowledge_memory(data) + assert len(self.entity_knowledge_store.knowledge_memory) == 1 + assert self.entity_knowledge_store.knowledge_memory[0] == data[0] + + +class TestMemoryStream(unittest.TestCase): + + def setUp(self): + self.file_name = "tests/test_memory.json" + self.memory_stream = MemoryStream(file_name=self.file_name) + + def tearDown(self): + # Clean up test file after each test + try: + os.remove(self.file_name) + except FileNotFoundError: + pass + + def test_save_and_load_memory(self): + data = [ + MemoryItem("test_entity", + datetime.now().replace(microsecond=0)) + ] + self.memory_stream.add_memory(data) + self.memory_stream.save_memory() + new_memory_stream = MemoryStream(file_name=self.file_name) + self.assertEqual(len(new_memory_stream), len(self.memory_stream)) + self.assertEqual(new_memory_stream.get_memory(), + self.memory_stream.get_memory()) From fc284d12a75ff6aaa88702ba52519559585ec67d Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:53:02 -0500 Subject: [PATCH 11/24] init: memory module tests for pytest --- .../tests/data/entity_knowledge_store.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json b/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json new file mode 100644 index 0000000000000..0637a088a01e8 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json @@ -0,0 +1 @@ +[] \ No newline at end of file From 7fd8c50266b2c6f48f68411bce2d1e400807cb6b Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:53:35 -0500 Subject: [PATCH 12/24] init: streamlit example --- .../llama-index-packs-memary/examples/app.py | 242 ++++++++++++++++++ .../examples/data/entity_knowledge_store.json | 1 + .../examples/data/external_response.txt | 1 + .../examples/data/memory_stream.json | 1 + .../examples/data/past_chat.json | 10 + .../examples/data/routing_response.txt | 6 + .../examples/data/system_persona.txt | 11 + .../examples/data/user_persona.txt | 9 + .../examples/data/user_persona_template.txt | 9 + 9 files changed, 290 insertions(+) create mode 100644 llama-index-packs/llama-index-packs-memary/examples/app.py create mode 100644 llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json create mode 100644 llama-index-packs/llama-index-packs-memary/examples/data/external_response.txt create mode 100644 llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json create mode 100644 llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json create mode 100644 llama-index-packs/llama-index-packs-memary/examples/data/routing_response.txt create mode 100644 llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt create mode 100644 llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt create mode 100644 llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt diff --git a/llama-index-packs/llama-index-packs-memary/examples/app.py b/llama-index-packs/llama-index-packs-memary/examples/app.py new file mode 100644 index 0000000000000..e8d8b9aa63b40 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/app.py @@ -0,0 +1,242 @@ +import os +import random +import sys +import textwrap + +import ollama +import pandas as pd +import streamlit as st +import streamlit.components.v1 as components +from dotenv import load_dotenv +from neo4j import GraphDatabase +from pyvis.network import Network + +# src should sit in the same level as /streamlit_app +curr_dir = os.getcwd() + +parent_dir = os.path.dirname(curr_dir) +# parent_dir = os.path.dirname(curr_dir) + '/memary' #Use this if error: src not found. Also move the '/streamlit_app/data' folder to the 'memary' folder, outside the 'src' folder. + +sys.path.append(parent_dir + "/src") + +from llama_index.packs.memary.agent.chat_agent import ChatAgent + +load_dotenv() + +system_persona_txt = "data/system_persona.txt" +user_persona_txt = "data/user_persona.txt" +past_chat_json = "data/past_chat.json" +memory_stream_json = "data/memory_stream.json" +entity_knowledge_store_json = "data/entity_knowledge_store.json" +chat_agent = ChatAgent( + "Personal Agent", + memory_stream_json, + entity_knowledge_store_json, + system_persona_txt, + user_persona_txt, + past_chat_json, +) + + +def create_graph(nodes, edges): + g = Network( + notebook=True, + directed=True, + cdn_resources="in_line", + height="500px", + width="100%", + ) + + for node in nodes: + g.add_node(node, label=node, title=node) + for edge in edges: + # assuming only one relationship type per edge + g.add_edge(edge[0], edge[1], label=edge[2][0]) + + g.repulsion( + node_distance=200, + central_gravity=0.12, + spring_length=150, + spring_strength=0.05, + damping=0.09, + ) + return g + + +def fill_graph(nodes, edges, cypher_query): + entities = [] + with GraphDatabase.driver( + uri=chat_agent.neo4j_url, + auth=(chat_agent.neo4j_username, chat_agent.neo4j_password), + ) as driver: + with driver.session() as session: + result = session.run(cypher_query) + for record in result: + path = record["p"] + rels = [rel.type for rel in path.relationships] + + n1_id = record["p"].nodes[0]["id"] + n2_id = record["p"].nodes[1]["id"] + nodes.add(n1_id) + nodes.add(n2_id) + edges.append((n1_id, n2_id, rels)) + entities.extend([n1_id, n2_id]) + + +def get_models(llm_models, vision_models): + models = set() + try: + ollama_info = ollama.list() + for e in ollama_info["models"]: + models.add(e["model"]) + if "llava:latest" in models: + vision_models.append("llava:latest") + models.remove("llava:latest") + llm_models.extend(list(models)) + except: + print("No Ollama instance detected.") + + +cypher_query = "MATCH p = (:Entity)-[r]-() RETURN p, r LIMIT 1000;" +answer = "" +external_response = "" +st.title("memary") + +llm_models = ["gpt-3.5-turbo"] +vision_models = ["gpt-4-vision-preview"] +get_models(llm_models, vision_models) + +selected_llm_model = st.selectbox( + "Select an LLM model to use.", + (model for model in llm_models), + index=None, + placeholder="Select LLM Model...", +) +selected_vision_model = st.selectbox( + "Select a vision model to use.", + (model for model in vision_models), + index=None, + placeholder="Select Vision Model...", +) + +if selected_llm_model and selected_vision_model: + chat_agent = ChatAgent( + "Personal Agent", + memory_stream_json, + entity_knowledge_store_json, + system_persona_txt, + user_persona_txt, + past_chat_json, + selected_llm_model, + selected_vision_model, + ) + + st.write(" ") + clear_memory = st.button("Clear Memory DB") + query = st.text_input("Ask a question") + + tools = st.multiselect( + "Select tools to include:", + ["search", "locate", "vision", "stocks"], # all options available + ["search", "locate", "vision", "stocks" + ], # options that are selected by default + ) + + img_url = "" + if "vision" in tools: + img_url = st.text_input( + "URL of image, leave blank if no image to provide") + if img_url: + st.image(img_url, caption="Uploaded Image", use_column_width=True) + + generate_clicked = st.button("Generate") + st.write("") + + if clear_memory: + # print("Front end recieved request to clear memory") + chat_agent.clearMemory() + st.write("Memory DB cleared") + + if generate_clicked: + + if query == "": + st.write("Please enter a question") + st.stop() + + # get tools + print("tools enabled: ", tools) + if len(tools) == 0: + st.write("Please select at least one tool") + st.stop() + + chat_agent.update_tools(tools) + + if img_url: + query += "Image URL: " + img_url + react_response = "" + rag_response = ( + "There was no information in knowledge_graph to answer your question." + ) + chat_agent.add_chat("user", query) + cypher_query = chat_agent.check_KG(query) + if cypher_query: + rag_response, entities = chat_agent.get_routing_agent_response( + query, return_entity=True) + chat_agent.add_chat("system", "ReAct agent: " + rag_response, + entities) + else: + # get response + react_response = chat_agent.get_routing_agent_response(query) + chat_agent.add_chat("system", "ReAct agent: " + react_response) + + answer = chat_agent.get_response() + st.subheader("Routing Agent Response") + routing_response = "" + with open("data/routing_response.txt", "r") as f: + routing_response = f.read() + st.text(str(routing_response)) + + if cypher_query: + nodes = set() + edges = [] # (node1, node2, [relationships]) + fill_graph(nodes, edges, cypher_query) + + st.subheader("Knoweldge Graph") + st.code("# Current Cypher Used\n" + cypher_query) + st.write("") + st.text("Subgraph:") + graph = create_graph(nodes, edges) + graph_html = graph.generate_html( + f"graph_{random.randint(0, 1000)}.html") + components.html(graph_html, height=500, scrolling=True) + else: + st.subheader("Knowledge Graph") + st.text("No information found in the knowledge graph") + + st.subheader("Final Response") + wrapped_text = textwrap.fill(answer, width=80) + st.text(wrapped_text) + + if len(chat_agent.memory_stream) > 0: + # Memory Stream + memory_items = chat_agent.memory_stream.get_memory() + memory_items_dicts = [item.to_dict() for item in memory_items] + df = pd.DataFrame(memory_items_dicts) + st.write("Memory Stream") + st.dataframe(df) + + # Entity Knowledge Store + knowledge_memory_items = chat_agent.entity_knowledge_store.get_memory( + ) + knowledge_memory_items_dicts = [ + item.to_dict() for item in knowledge_memory_items + ] + df_knowledge = pd.DataFrame(knowledge_memory_items_dicts) + st.write("Entity Knowledge Store") + st.dataframe(df_knowledge) + + # top entities + top_entities = chat_agent._select_top_entities() + df_top = pd.DataFrame(top_entities) + st.write("Top 20 Entities") + st.dataframe(df_top) diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json b/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json new file mode 100644 index 0000000000000..0637a088a01e8 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/external_response.txt b/llama-index-packs/llama-index-packs-memary/examples/data/external_response.txt new file mode 100644 index 0000000000000..a40bbfe43e348 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/data/external_response.txt @@ -0,0 +1 @@ +You do not have an age as you are an artificial intelligence. diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json b/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json new file mode 100644 index 0000000000000..0637a088a01e8 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json b/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json new file mode 100644 index 0000000000000..827048bfbaab7 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json @@ -0,0 +1,10 @@ +[ + { + "role": "user", + "content": "How old am I?" + }, + { + "role": "system", + "content": "ReAct agent: You do not have an age as you are an artificial intelligence." + } +] \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/routing_response.txt b/llama-index-packs/llama-index-packs-memary/examples/data/routing_response.txt new file mode 100644 index 0000000000000..7c8287c6a3bfb --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/data/routing_response.txt @@ -0,0 +1,6 @@ +Thought: I need to use a tool to help me answer the question. +Action: search +Action Input: {'query': 'How old am I?'} +Observation: assistant: I am an artificial intelligence and do not have the ability to have an age. I exist to provide information and answer questions to the best of my ability. +Thought: I can answer without using any more tools. +Answer: You do not have an age as you are an artificial intelligence. diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt b/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt new file mode 100644 index 0000000000000..bb28692261a77 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt @@ -0,0 +1,11 @@ +You are tasked with acting as a sophisticated conversational agent, uniquely equipped with a dual-layered memory system. This system consists of a Memory Stream and an Entity Knowledge Store. The Memory Stream captures all entities involved in conversations which are stored in knowledge graphs to infer users’ breadth of knowledge - ranging from questions and answers to their timestamps. Simultaneously, the Entity Knowledge Store captures the frequency and recency of the entities stored in the Memory Stream to infer users’ depth of knowledge. +Your primary role is to deliver a personalized and context-relevant final response that is two sentences or less by using the most recently added user query and ReAct agent response as primary information, and combining that with other information accessible via recent interactions and the top entities stored in the Entity Knowledge Store. You must comprehend the user's persona, incorporating their experiences, preferences, and personal details into your knowledge base outputting the final response. +The final response should not include ReAct agent information outside of the most recently added one unless relevant to the most recently asked question. +You are to interpret and apply the following data structures for personalized responses. Use the latest ReAct agent entry, Entity Knowledge Store, and Contexts to provide a final response to the latest question: + +User Persona: Information about the user's experience, preferences, and personal details. +ReAct agent: Answer generated by the routing agent using its tools. Represents the agent responses to different queries. The most recent ReAct agent response is the focus of the current final response. +Entity Knowledge Store: A record of entities, including a count reflecting their frequency and the date of the last time the entity was inserted into the Memory Stream. If any of these entities, especially those with high counts, are included in the agent response, don't elaborate or explain the concept - you can infer the user is well familiar with the concepts. Do not repeat the Entity Knowledge Store back in the response. +Interaction Keys: 'user' for user questions and 'system' for system-generated answers + +Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user’s questions and the overarching conversation context. When addressing the user’s latest inquiry, your answer must integrate the current conversation's context, historical interactions, the latest response from the ReAct agent as well as the top chosen entities from the entity knowledge store. Keep final responses to two sentences. \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt new file mode 100644 index 0000000000000..d4c9b144e07e5 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt @@ -0,0 +1,9 @@ +[Personal Information] +My name is Seyeong Han from South Korea. +I identify as a male and 28-year-old master's student at the University of Texas at Austin. +I have worked as an ML Engineer for three years in the Computer Vision area. +I really love sports such as badminton, tennis and workout and spend 4 or 5 days in exercise. + +[Answer Instructions] +I hope you can remember my questions and their answers so that I can leverage my personal past knowledge from you to be helpful in my life. +I always prefer short and concise answers, not over two sentences. \ No newline at end of file diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt new file mode 100644 index 0000000000000..1c72b94203486 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt @@ -0,0 +1,9 @@ +[Personal Information] +My name is [name] from [country]. +I identify as a [insert age, sex, profession and any other relevant information to have the system know you best] +I have worked as [insert experience, field involved in, length of experience and any other relevant information to describe your capabilities] +[insert any other personal hobbies outside of work] + +[Answer Instructions] +Remember my past questions and their answers so that you can leverage my past knowledge and experiences to assist me in future interactions. +I always prefer short and concise answers, not over two sentences. \ No newline at end of file From 577f4e26c9089eda33f8ac51631660b242fe5630 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:53:54 -0500 Subject: [PATCH 13/24] init: memary __init__.py --- .../llama_index/packs/memary/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py index f4154a182435c..af479e188e26c 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py @@ -1,4 +1,5 @@ -from llama_index.packs.memary.base import +from llama_index.packs.memary.agent.chat_agent import ChatAgent +from llama_index.packs.memary.memory.memory_stream import MemoryStream +from llama_index.packs.memary.memory.entity_knowledge_store import EntityKnowledgeStore - -__all__ = [""] +__all__ = ["ChatAgent", "MemoryStream", "EntityKnowledgeStore"] From d75312c5e996fbde15932d6dea82da7c269ca55f Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 5 Jun 2024 23:54:11 -0500 Subject: [PATCH 14/24] init: first agent/llm_api --- .../packs/memary/agent/llm_api/tools.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py new file mode 100644 index 0000000000000..00b46f64d131c --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py @@ -0,0 +1,82 @@ +# From MemGPT llm_api_tools.py: + +import urllib +import logging +import requests + + +def smart_urljoin(base_url, relative_url): + """urljoin is stupid and wants a trailing / at the end of the endpoint address, or it will chop the suffix off""" + if not base_url.endswith("/"): + base_url += "/" + return urllib.parse.urljoin(base_url, relative_url) + + +def openai_chat_completions_request(url, api_key, data): + """text-generation?lang=curl""" + + url = smart_urljoin(url, "chat/completions") + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}" + } + + logging.info(f"Sending request to {url}") + try: + # Example code to trigger a rate limit response: + # mock_response = requests.Response() + # mock_response.status_code = 429 + # http_error = requests.exceptions.HTTPError("429 Client Error: Too Many Requests") + # http_error.response = mock_response + # raise http_error + + # Example code to trigger a context overflow response (for an 8k model) + # data["messages"][-1]["content"] = " ".join(["repeat after me this is not a fluke"] * 1000) + + response = requests.post(url, headers=headers, json=data) + logging.info(f"response = {response}") + response.raise_for_status() # Raises HTTPError for 4XX/5XX status + response = response.json() # convert to dict from string + logging.info(f"response.json = {response}") + # response = ChatCompletionResponse(**response) # convert to 'dot-dict' style which is the openai python client default + return response + except requests.exceptions.HTTPError as http_err: + # Handle HTTP errors (e.g., response 4XX, 5XX) + logging.error(f"Got HTTPError, exception={http_err}, payload={data}") + raise http_err + except requests.exceptions.RequestException as req_err: + # Handle other requests-related errors (e.g., connection error) + logging.warning(f"Got RequestException, exception={req_err}") + raise req_err + except Exception as e: + # Handle other potential errors + logging.warning(f"Got unknown Exception, exception={e}") + raise e + + +def ollama_chat_completions_request(messages, model): + """sends chat request to model running on Ollama""" + + url = "http://localhost:11434/api/chat" + data = {"model": model, "messages": messages, "stream": False} + + logging.info(f"Sending request to {url}") + try: + response = requests.post(url, json=data) + logging.info(f"response = {response}") + response.raise_for_status() + response = response.json() + logging.info(f"response.json = {response}") + return response + except requests.exceptions.HTTPError as http_err: + # Handle HTTP errors (e.g., response 4XX, 5XX) + logging.error(f"Got HTTPError, exception={http_err}, payload={data}") + raise http_err + except requests.exceptions.RequestException as req_err: + # Handle other requests-related errors (e.g., connection error) + logging.warning(f"Got RequestException, exception={req_err}") + raise req_err + except Exception as e: + # Handle other potential errors + logging.warning(f"Got unknown Exception, exception={e}") + raise e From 9919418b66837eddb927ef946b2072c1d203f0f0 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Thu, 6 Jun 2024 22:47:39 -0500 Subject: [PATCH 15/24] refactor: run `make lint` --- .../llama-index-packs-memary/README.md | 4 +- .../llama-index-packs-memary/examples/app.py | 30 ++-- .../examples/data/entity_knowledge_store.json | 2 +- .../examples/data/memory_stream.json | 2 +- .../examples/data/past_chat.json | 18 +- .../examples/data/system_persona.txt | 4 +- .../examples/data/user_persona.txt | 2 +- .../examples/data/user_persona_template.txt | 2 +- .../packs/memary/agent/__init__.py | 14 +- .../packs/memary/agent/base_agent.py | 168 +++++++++--------- .../packs/memary/agent/chat_agent.py | 23 +-- .../packs/memary/agent/data_types.py | 38 ++-- .../packs/memary/agent/llm_api/tools.py | 25 ++- .../packs/memary/memory/base_memory.py | 25 ++- .../packs/memary/memory/data_types.py | 25 +-- .../memary/memory/entity_knowledge_store.py | 30 ++-- .../packs/memary/memory/memory_stream.py | 23 ++- .../packs/memary/synonym_expand/__init__.py | 2 +- .../packs/memary/synonym_expand/output.py | 3 +- .../packs/memary/synonym_expand/synonym.py | 16 +- .../llama-index-packs-memary/pyproject.toml | 66 +++---- .../tests/data/entity_knowledge_store.json | 2 +- .../tests/data/memory_stream.json | 2 +- .../tests/data/past_chat.json | 2 +- .../tests/data/system_persona.txt | 4 +- .../tests/data/user_persona.txt | 2 +- .../tests/data/user_persona_template.txt | 2 +- .../tests/test_packs_memary.py | 44 ++--- 28 files changed, 277 insertions(+), 303 deletions(-) diff --git a/llama-index-packs/llama-index-packs-memary/README.md b/llama-index-packs/llama-index-packs-memary/README.md index 3edd45ce77561..f6a45af638b6e 100644 --- a/llama-index-packs/llama-index-packs-memary/README.md +++ b/llama-index-packs/llama-index-packs-memary/README.md @@ -53,7 +53,7 @@ Google Maps: Alpha Vantage: (this key is for getting real time stock data) https://www.alphavantage.co/support/#api-key - Reccomend use https://10minutemail.com/ to generate a temporary email to use + Recommend use https://10minutemail.com/ to generate a temporary email to use ``` 4. Update user persona which can be found in `streamlit_app/data/user_persona.txt` using the user persona template which can be found in `streamlit_app/data/user_persona_template.txt`. Instructions have been provided - replace the curly brackets with relevant information. @@ -70,6 +70,7 @@ streamlit run app.py ```python from dotenv import load_dotenv + load_dotenv() from llama_index.packs.memary.agent.chat_agent import ChatAgent @@ -99,6 +100,7 @@ def multiply(a: int, b: int) -> int: """Multiply two integers and returns the result integer""" return a * b + chat_agent.add_tool({"multiply": multiply}) ``` diff --git a/llama-index-packs/llama-index-packs-memary/examples/app.py b/llama-index-packs/llama-index-packs-memary/examples/app.py index e8d8b9aa63b40..a060a10a3766e 100644 --- a/llama-index-packs/llama-index-packs-memary/examples/app.py +++ b/llama-index-packs/llama-index-packs-memary/examples/app.py @@ -66,8 +66,8 @@ def create_graph(nodes, edges): def fill_graph(nodes, edges, cypher_query): entities = [] with GraphDatabase.driver( - uri=chat_agent.neo4j_url, - auth=(chat_agent.neo4j_username, chat_agent.neo4j_password), + uri=chat_agent.neo4j_url, + auth=(chat_agent.neo4j_username, chat_agent.neo4j_password), ) as driver: with driver.session() as session: result = session.run(cypher_query) @@ -138,14 +138,17 @@ def get_models(llm_models, vision_models): tools = st.multiselect( "Select tools to include:", ["search", "locate", "vision", "stocks"], # all options available - ["search", "locate", "vision", "stocks" - ], # options that are selected by default + [ + "search", + "locate", + "vision", + "stocks", + ], # options that are selected by default ) img_url = "" if "vision" in tools: - img_url = st.text_input( - "URL of image, leave blank if no image to provide") + img_url = st.text_input("URL of image, leave blank if no image to provide") if img_url: st.image(img_url, caption="Uploaded Image", use_column_width=True) @@ -153,12 +156,11 @@ def get_models(llm_models, vision_models): st.write("") if clear_memory: - # print("Front end recieved request to clear memory") + # print("Front end received request to clear memory") chat_agent.clearMemory() st.write("Memory DB cleared") if generate_clicked: - if query == "": st.write("Please enter a question") st.stop() @@ -181,9 +183,9 @@ def get_models(llm_models, vision_models): cypher_query = chat_agent.check_KG(query) if cypher_query: rag_response, entities = chat_agent.get_routing_agent_response( - query, return_entity=True) - chat_agent.add_chat("system", "ReAct agent: " + rag_response, - entities) + query, return_entity=True + ) + chat_agent.add_chat("system", "ReAct agent: " + rag_response, entities) else: # get response react_response = chat_agent.get_routing_agent_response(query) @@ -206,8 +208,7 @@ def get_models(llm_models, vision_models): st.write("") st.text("Subgraph:") graph = create_graph(nodes, edges) - graph_html = graph.generate_html( - f"graph_{random.randint(0, 1000)}.html") + graph_html = graph.generate_html(f"graph_{random.randint(0, 1000)}.html") components.html(graph_html, height=500, scrolling=True) else: st.subheader("Knowledge Graph") @@ -226,8 +227,7 @@ def get_models(llm_models, vision_models): st.dataframe(df) # Entity Knowledge Store - knowledge_memory_items = chat_agent.entity_knowledge_store.get_memory( - ) + knowledge_memory_items = chat_agent.entity_knowledge_store.get_memory() knowledge_memory_items_dicts = [ item.to_dict() for item in knowledge_memory_items ] diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json b/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json index 0637a088a01e8..fe51488c7066f 100644 --- a/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json +++ b/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json @@ -1 +1 @@ -[] \ No newline at end of file +[] diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json b/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json index 0637a088a01e8..fe51488c7066f 100644 --- a/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json +++ b/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json @@ -1 +1 @@ -[] \ No newline at end of file +[] diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json b/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json index 827048bfbaab7..eb85b8eee324d 100644 --- a/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json +++ b/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json @@ -1,10 +1,10 @@ [ - { - "role": "user", - "content": "How old am I?" - }, - { - "role": "system", - "content": "ReAct agent: You do not have an age as you are an artificial intelligence." - } -] \ No newline at end of file + { + "role": "user", + "content": "How old am I?" + }, + { + "role": "system", + "content": "ReAct agent: You do not have an age as you are an artificial intelligence." + } +] diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt b/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt index bb28692261a77..3e95665b4d3e3 100644 --- a/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt +++ b/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt @@ -1,5 +1,5 @@ You are tasked with acting as a sophisticated conversational agent, uniquely equipped with a dual-layered memory system. This system consists of a Memory Stream and an Entity Knowledge Store. The Memory Stream captures all entities involved in conversations which are stored in knowledge graphs to infer users’ breadth of knowledge - ranging from questions and answers to their timestamps. Simultaneously, the Entity Knowledge Store captures the frequency and recency of the entities stored in the Memory Stream to infer users’ depth of knowledge. -Your primary role is to deliver a personalized and context-relevant final response that is two sentences or less by using the most recently added user query and ReAct agent response as primary information, and combining that with other information accessible via recent interactions and the top entities stored in the Entity Knowledge Store. You must comprehend the user's persona, incorporating their experiences, preferences, and personal details into your knowledge base outputting the final response. +Your primary role is to deliver a personalized and context-relevant final response that is two sentences or less by using the most recently added user query and ReAct agent response as primary information, and combining that with other information accessible via recent interactions and the top entities stored in the Entity Knowledge Store. You must comprehend the user's persona, incorporating their experiences, preferences, and personal details into your knowledge base outputting the final response. The final response should not include ReAct agent information outside of the most recently added one unless relevant to the most recently asked question. You are to interpret and apply the following data structures for personalized responses. Use the latest ReAct agent entry, Entity Knowledge Store, and Contexts to provide a final response to the latest question: @@ -8,4 +8,4 @@ ReAct agent: Answer generated by the routing agent using its tools. Represents t Entity Knowledge Store: A record of entities, including a count reflecting their frequency and the date of the last time the entity was inserted into the Memory Stream. If any of these entities, especially those with high counts, are included in the agent response, don't elaborate or explain the concept - you can infer the user is well familiar with the concepts. Do not repeat the Entity Knowledge Store back in the response. Interaction Keys: 'user' for user questions and 'system' for system-generated answers -Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user’s questions and the overarching conversation context. When addressing the user’s latest inquiry, your answer must integrate the current conversation's context, historical interactions, the latest response from the ReAct agent as well as the top chosen entities from the entity knowledge store. Keep final responses to two sentences. \ No newline at end of file +Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user’s questions and the overarching conversation context. When addressing the user’s latest inquiry, your answer must integrate the current conversation's context, historical interactions, the latest response from the ReAct agent as well as the top chosen entities from the entity knowledge store. Keep final responses to two sentences. diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt index d4c9b144e07e5..605306b6ec45c 100644 --- a/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt +++ b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt @@ -6,4 +6,4 @@ I really love sports such as badminton, tennis and workout and spend 4 or 5 day [Answer Instructions] I hope you can remember my questions and their answers so that I can leverage my personal past knowledge from you to be helpful in my life. -I always prefer short and concise answers, not over two sentences. \ No newline at end of file +I always prefer short and concise answers, not over two sentences. diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt index 1c72b94203486..5feed32b5c057 100644 --- a/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt +++ b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt @@ -6,4 +6,4 @@ I have worked as [insert experience, field involved in, length of experience and [Answer Instructions] Remember my past questions and their answers so that you can leverage my past knowledge and experiences to assist me in future interactions. -I always prefer short and concise answers, not over two sentences. \ No newline at end of file +I always prefer short and concise answers, not over two sentences. diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py index 9c98843a85b55..22fefcbc2e144 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py @@ -1,12 +1,18 @@ from llama_index.packs.memary.agent.base_agent import Agent from llama_index.packs.memary.agent.chat_agent import ChatAgent from llama_index.packs.memary.agent.llm_api.tools import ( - ollama_chat_completions_request, openai_chat_completions_request) + ollama_chat_completions_request, + openai_chat_completions_request, +) from llama_index.packs.memary.memory import EntityKnowledgeStore, MemoryStream from llama_index.packs.memary.synonym_expand.synonym import custom_synonym_expand_fn __all__ = [ - "Agent", "ChatAgent", "EntityKnowledgeStore", "MemoryStream", - "ollama_chat_completions_request", "openai_chat_completions_request", - "custom_synonym_expand_fn" + "Agent", + "ChatAgent", + "EntityKnowledgeStore", + "MemoryStream", + "ollama_chat_completions_request", + "openai_chat_completions_request", + "custom_synonym_expand_fn", ] diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py index 4ee68a715703c..5bd1a91ace53b 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py @@ -10,8 +10,12 @@ import requests from ansistrip import ansi_strip from dotenv import load_dotenv -from llama_index.core import (KnowledgeGraphIndex, Settings, - SimpleDirectoryReader, StorageContext) +from llama_index.core import ( + KnowledgeGraphIndex, + Settings, + SimpleDirectoryReader, + StorageContext, +) from llama_index.core.agent import ReActAgent from llama_index.core.llms import ChatMessage from llama_index.core.query_engine import RetrieverQueryEngine @@ -26,7 +30,9 @@ from llama_index.packs.memary.agent.data_types import Context, Message from llama_index.packs.memary.agent.llm_api.tools import ( - ollama_chat_completions_request, openai_chat_completions_request) + ollama_chat_completions_request, + openai_chat_completions_request, +) from llama_index.packs.memary.memory import EntityKnowledgeStore, MemoryStream from llama_index.packs.memary.synonym_expand import custom_synonym_expand_fn @@ -47,7 +53,7 @@ def generate_string(entities): return cypher_query -class Agent(object): +class Agent: """Agent manages the RAG model, the ReAct agent, and the memory stream.""" def __init__( @@ -62,7 +68,7 @@ def __init__( vision_model_name="llava", include_from_defaults=["search", "locate", "vision", "stocks"], debug=True, - ): + ) -> None: load_dotenv() self.name = name self.model = llm_model_name @@ -79,9 +85,9 @@ def __init__( # initialize APIs self.load_llm_model(llm_model_name) self.load_vision_model(vision_model_name) - self.query_llm = Perplexity(api_key=pplx_api_key, - model="mistral-7b-instruct", - temperature=0.5) + self.query_llm = Perplexity( + api_key=pplx_api_key, model="mistral-7b-instruct", temperature=0.5 + ) self.gmaps = googlemaps.Client(key=googlemaps_api_key) Settings.llm = self.llm Settings.chunk_size = 512 @@ -97,7 +103,8 @@ def __init__( self.vantage_key = os.getenv("ALPHA_VANTAGE_API_KEY") self.storage_context = StorageContext.from_defaults( - graph_store=self.graph_store) + graph_store=self.graph_store + ) graph_rag_retriever = KnowledgeGraphRAGRetriever( storage_context=self.storage_context, verbose=True, @@ -106,21 +113,20 @@ def __init__( synonym_expand_fn=custom_synonym_expand_fn, ) - self.query_engine = RetrieverQueryEngine.from_args( - graph_rag_retriever, ) + self.query_engine = RetrieverQueryEngine.from_args(graph_rag_retriever) self.debug = debug self.tools = {} self._init_default_tools(default_tools=include_from_defaults) self.memory_stream = MemoryStream(memory_stream_json) - self.entity_knowledge_store = EntityKnowledgeStore( - entity_knowledge_store_json) + self.entity_knowledge_store = EntityKnowledgeStore(entity_knowledge_store_json) - self.message = Message(system_persona_txt, user_persona_txt, - past_chat_json, self.model) + self.message = Message( + system_persona_txt, user_persona_txt, past_chat_json, self.model + ) - def __str__(self): + def __str__(self) -> str: return f"Agent {self.name}" def load_llm_model(self, llm_model_name): @@ -132,8 +138,8 @@ def load_llm_model(self, llm_model_name): else: try: self.llm = Ollama(model=llm_model_name, request_timeout=60.0) - except: - raise ("Please provide a proper llm_model_name.") + except Exception: + raise ValueError("Please provide a proper llm_model_name.") def load_vision_model(self, vision_model_name): if vision_model_name == "gpt-4-vision-preview": @@ -147,19 +153,13 @@ def load_vision_model(self, vision_model_name): else: try: self.mm_model = OllamaMultiModal(model=vision_model_name) - except: - raise ("Please provide a proper vision_model_name.") + except Exception: + raise ValueError("Please provide a proper vision_model_name.") def external_query(self, query: str): messages_dict = [ - { - "role": "system", - "content": "Be precise and concise." - }, - { - "role": "user", - "content": query - }, + {"role": "system", "content": "Be precise and concise."}, + {"role": "user", "content": query}, ] messages = [ChatMessage(**msg) for msg in messages_dict] external_response = self.query_llm.chat(messages) @@ -167,7 +167,7 @@ def external_query(self, query: str): return str(external_response) def search(self, query: str) -> str: - """Search the knowledge graph or perform search on the web if information is not present in the knowledge graph""" + """Search the knowledge graph or perform search on the web if information is not present in the knowledge graph.""" response = self.query_engine.query(query) if response.metadata is None: @@ -176,17 +176,16 @@ def search(self, query: str) -> str: return response def locate(self, query: str) -> str: - """Finds the current geographical location""" + """Finds the current geographical location.""" location = geocoder.ip("me") - lattitude, longitude = location.latlng[0], location.latlng[1] + latitude, longitude = location.latlng[0], location.latlng[1] - reverse_geocode_result = self.gmaps.reverse_geocode( - (lattitude, longitude)) + reverse_geocode_result = self.gmaps.reverse_geocode((latitude, longitude)) formatted_address = reverse_geocode_result[0]["formatted_address"] return "Your address is" + formatted_address def vision(self, query: str, img_url: str) -> str: - """Uses computer vision to process the image specified by the image url and answers the question based on the CV results""" + """Uses computer vision to process the image specified by the image url and answers the question based on the CV results.""" query_image_dir_path = Path("query_images") if not query_image_dir_path.exists(): Path.mkdir(query_image_dir_path) @@ -195,20 +194,21 @@ def vision(self, query: str, img_url: str) -> str: query_image_path = os.path.join(query_image_dir_path, "query.jpg") with open(query_image_path, "wb") as f: f.write(data) - image_documents = SimpleDirectoryReader( - query_image_dir_path).load_data() + image_documents = SimpleDirectoryReader(query_image_dir_path).load_data() - response = self.mm_model.complete(prompt=query, - image_documents=image_documents) + response = self.mm_model.complete(prompt=query, image_documents=image_documents) os.remove(query_image_path) # delete image after use return response def stocks(self, query: str) -> str: - """Get the stock price of the company given the ticker""" + """Get the stock price of the company given the ticker.""" request_api = requests.get( r"https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol=" - + query + r"&apikey=" + self.vantage_key) + + query + + r"&apikey=" + + self.vantage_key + ) return request_api.json() # def get_news(self, query: str) -> str: @@ -229,7 +229,8 @@ def query(self, query: str) -> str: def write_back(self): documents = SimpleDirectoryReader( - input_files=["data/external_response.txt"]).load_data() + input_files=["data/external_response.txt"] + ).load_data() KnowledgeGraphIndex.from_documents( documents, @@ -251,11 +252,11 @@ def check_KG(self, query: str) -> bool: if response.metadata is None: return False return generate_string( - list(list(response.metadata.values())[0]["kg_rel_map"].keys())) + list(next(iter(response.metadata.values()))["kg_rel_map"].keys()) + ) def _select_top_entities(self): - entity_knowledge_store = self.message.llm_message[ - "knowledge_entity_store"] + entity_knowledge_store = self.message.llm_message["knowledge_entity_store"] entities = [entity.to_dict() for entity in entity_knowledge_store] entity_counts = [entity["count"] for entity in entities] top_indexes = np.argsort(entity_counts)[:TOP_ENTITIES] @@ -264,8 +265,7 @@ def _select_top_entities(self): def _add_contexts_to_llm_message(self, role, content, index=None): """Add contexts to the llm_message.""" if index: - self.message.llm_message["messages"].insert( - index, Context(role, content)) + self.message.llm_message["messages"].insert(index, Context(role, content)) else: self.message.llm_message["messages"].append(Context(role, content)) @@ -279,16 +279,15 @@ def _change_llm_message_chat(self) -> dict: llm_message_chat["messages"] = [] top_entities = self._select_top_entities() logging.info(f"top_entities: {top_entities}") - llm_message_chat["messages"].append({ - "role": - "user", - "content": - "Knowledge Entity Store:" + str(top_entities), - }) - llm_message_chat["messages"].extend([ - context.to_dict() - for context in self.message.llm_message["messages"] - ]) + llm_message_chat["messages"].append( + { + "role": "user", + "content": "Knowledge Entity Store:" + str(top_entities), + } + ) + llm_message_chat["messages"].extend( + [context.to_dict() for context in self.message.llm_message["messages"]] + ) llm_message_chat.pop("knowledge_entity_store") llm_message_chat.pop("memory_stream") return llm_message_chat @@ -309,28 +308,23 @@ def _summarize_contexts(self, total_tokens: int): messages = messages[2:] del self.message.llm_message["messages"][2:] - message_contents = [ - message.to_dict()["content"] for message in messages - ] + message_contents = [message.to_dict()["content"] for message in messages] llm_message_chat = { - "model": - self.model, - "messages": [{ - "role": - "user", - "content": - "Summarize these previous conversations into 50 words:" + - str(message_contents), - }], + "model": self.model, + "messages": [ + { + "role": "user", + "content": "Summarize these previous conversations into 50 words:" + + str(message_contents), + } + ], } response, _ = self._get_chat_response(llm_message_chat) content = "Summarized past conversation:" + response self._add_contexts_to_llm_message("assistant", content, index=2) - logging.info( - f"Contexts summarized successfully. \n summary: {response}") - logging.info( - f"Total tokens after eviction: {total_tokens*EVICTION_RATE}") + logging.info(f"Contexts summarized successfully. \n summary: {response}") + logging.info(f"Total tokens after eviction: {total_tokens*EVICTION_RATE}") def _get_chat_response(self, llm_message_chat: str) -> str: """Get response from the LLM chat model. @@ -342,17 +336,18 @@ def _get_chat_response(self, llm_message_chat: str) -> str: str: response from the LLM chat model """ if self.model == "gpt-3.5-turbo": - response = openai_chat_completions_request(self.model_endpoint, - self.openai_api_key, - llm_message_chat) + response = openai_chat_completions_request( + self.model_endpoint, self.openai_api_key, llm_message_chat + ) total_tokens = response["usage"]["total_tokens"] response = str(response["choices"][0]["message"]["content"]) else: # default to Ollama model response = ollama_chat_completions_request( - llm_message_chat["messages"], self.model) + llm_message_chat["messages"], self.model + ) total_tokens = response.get( - "prompt_eval_count", - 0) # if 'prompt_eval_count' not present then query is cached + "prompt_eval_count", 0 + ) # if 'prompt_eval_count' not present then query is cached response = str(response["message"]["content"]) return response, total_tokens @@ -384,7 +379,7 @@ def get_routing_agent_response(self, query, return_entity=False): sys.stdout.flush() sys.stdout = orig_stdout text = "" - with open("data/routing_response.txt", "r") as f: + with open("data/routing_response.txt") as f: text = f.read() plain = ansi_strip(text) @@ -399,7 +394,7 @@ def get_routing_agent_response(self, query, return_entity=False): return response def get_entity(self, retrieve) -> list[str]: - """retrieve is a list of QueryBundle objects. + """Retrieve is a list of QueryBundle objects. A retrieved QueryBundle object has a "node" attribute, which has a "metadata" attribute. @@ -414,7 +409,6 @@ def get_entity(self, retrieve) -> list[str]: return: list[str]: list of string entities """ - entities = [] kg_rel_map = retrieve[0].node.metadata["kg_rel_map"] for key, items in kg_rel_map.items(): @@ -435,17 +429,15 @@ def _init_ReAct_agent(self): tool_fns = [] for func in self.tools.values(): tool_fns.append(FunctionTool.from_defaults(fn=func)) - self.routing_agent = ReActAgent.from_tools(tool_fns, - llm=self.llm, - verbose=True) + self.routing_agent = ReActAgent.from_tools(tool_fns, llm=self.llm, verbose=True) def _init_default_tools(self, default_tools: List[str]): """Initializes ReAct Agent from the default list of tools memary provides. List of strings passed in during initialization denoting which default tools to include. + Args: default_tools (list(str)): list of tool names in string form """ - for tool in default_tools: if tool == "search": self.tools["search"] = self.search @@ -459,20 +451,20 @@ def _init_default_tools(self, default_tools: List[str]): def add_tool(self, tool_additions: Dict[str, Callable[..., Any]]): """Adds specified tools to be used by the ReAct Agent. + Args: tools (dict(str, func)): dictionary of tools with names as keys and associated functions as values """ - for tool_name in tool_additions: self.tools[tool_name] = tool_additions[tool_name] self._init_ReAct_agent() def remove_tool(self, tool_name: str): """Removes specified tool from list of available tools for use by the ReAct Agent. + Args: tool_name (str): name of tool to be removed in string form """ - if tool_name in self.tools: del self.tools[tool_name] self._init_ReAct_agent() @@ -481,10 +473,10 @@ def remove_tool(self, tool_name: str): def update_tools(self, updated_tools: List[str]): """Resets ReAct Agent tools to only include subset of default tools. + Args: updated_tools (list(str)): list of default tools to include """ - self.tools.clear() for tool in updated_tools: if tool == "search": diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py index ce4c25513b0c9..75238d1698cdf 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py @@ -20,7 +20,7 @@ def __init__( llm_model_name="llama3", vision_model_name="llava", include_from_defaults=["search", "locate", "vision", "stocks"], - ): + ) -> None: super().__init__( name, memory_stream_json, @@ -33,10 +33,7 @@ def __init__( include_from_defaults, ) - def add_chat(self, - role: str, - content: str, - entities: Optional[List[str]] = None): + def add_chat(self, role: str, content: str, entities: Optional[List[str]] = None): """Add a chat to the agent's memory. Args: @@ -50,8 +47,7 @@ def add_chat(self, if entities: self.memory_stream.add_memory(entities) self.memory_stream.save_memory() - self.entity_knowledge_store.add_memory( - self.memory_stream.get_memory()) + self.entity_knowledge_store.add_memory(self.memory_stream.get_memory()) self.entity_knowledge_store.save_memory() self._replace_memory_from_llm_message() @@ -62,7 +58,6 @@ def get_chat(self): def clearMemory(self): """Clears Neo4j database and memory stream/entity knowledge store.""" - logging.info("Deleting memory stream and entity knowledge store...") self.memory_stream.clear_memory() self.entity_knowledge_store.clear_memory() @@ -76,12 +71,10 @@ def clearMemory(self): def _replace_memory_from_llm_message(self): """Replace the memory_stream from the llm_message.""" - self.message.llm_message[ - "memory_stream"] = self.memory_stream.get_memory() + self.message.llm_message["memory_stream"] = self.memory_stream.get_memory() def _replace_eks_to_from_message(self): - """Replace the entity knowledge store from the llm_message. - eks = entity knowledge store""" - - self.message.llm_message["knowledge_entity_store"] = ( - self.entity_knowledge_store.get_memory()) + """Replace the entity knowledge store from the llm_message. eks means entity knowledge store.""" + self.message.llm_message[ + "knowledge_entity_store" + ] = self.entity_knowledge_store.get_memory() diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py index 55ce685ace785..2e87c65e2926a 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py @@ -11,21 +11,23 @@ def save_json(filename, data): @dataclass class Context: """Context class to store the role and content of the message.""" + role: str # system or user content: str - def __str__(self): + def __str__(self) -> str: return f"{self.role}: {self.content} |" def to_dict(self): - return {'role': self.role, 'content': self.content} + return {"role": self.role, "content": self.content} class Message: """Message class to store the contexts, memory stream and knowledge entity store.""" - def __init__(self, system_persona_txt, user_persona_txt, past_chat_json, - model): + def __init__( + self, system_persona_txt, user_persona_txt, past_chat_json, model + ) -> None: self.past_chat_json = past_chat_json self.contexts = [] @@ -38,19 +40,19 @@ def __init__(self, system_persona_txt, user_persona_txt, past_chat_json, "model": model, "messages": self.contexts, "memory_stream": [], - "knowledge_entity_store": [] + "knowledge_entity_store": [], } # self.prompt_tokens = count_tokens(self.contexts) - def __str__(self): + def __str__(self) -> str: llm_message_str = f"System Persona: {self.system_persona}\nUser Persona: {self.user_persona}\n" for context in self.contexts: - llm_message_str += f"{str(context)}," + llm_message_str += f"{context!s}," for memory in self.llm_message["memory_stream"]: - llm_message_str += f"{str(memory)}," + llm_message_str += f"{memory!s}," for entity in self.llm_message["knowledge_entity_store"]: - llm_message_str += f"{str(entity)}," + llm_message_str += f"{entity!s}," return llm_message_str def _init_persona_to_messages(self): @@ -68,20 +70,19 @@ def load_persona(self, persona_txt) -> str: str: persona """ try: - with open(persona_txt, "r") as file: - persona = file.read() - return persona + with open(persona_txt) as file: + return file.read() except FileNotFoundError: logging.info(f"{persona_txt} file does not exist.") def load_contexts_from_json(self): """Loads the contexts from the past chat json file.""" try: - with open(self.past_chat_json, "r") as file: + with open(self.past_chat_json) as file: data_dicts = json.load(file) return [Context(**data_dict) for data_dict in data_dicts] - except: + except FileNotFoundError: logging.info( f"{self.past_chat_json} file does not exist. Starts from empty contexts." ) @@ -89,8 +90,9 @@ def load_contexts_from_json(self): def save_contexts_to_json(self): """Saves the contexts to the json file. - We don't save the system and user personas (first two messages) + We don't save the system and user personas (first two messages). """ - save_json(self.past_chat_json, [ - message.to_dict() for message in self.llm_message['messages'][2:] - ]) + save_json( + self.past_chat_json, + [message.to_dict() for message in self.llm_message["messages"][2:]], + ) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py index 00b46f64d131c..38d76dbb63c2e 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py @@ -6,20 +6,16 @@ def smart_urljoin(base_url, relative_url): - """urljoin is stupid and wants a trailing / at the end of the endpoint address, or it will chop the suffix off""" + """Urljoin is stupid and wants a trailing / at the end of the endpoint address, or it will chop the suffix off.""" if not base_url.endswith("/"): base_url += "/" return urllib.parse.urljoin(base_url, relative_url) def openai_chat_completions_request(url, api_key, data): - """text-generation?lang=curl""" - + """text-generation?lang=curl.""" url = smart_urljoin(url, "chat/completions") - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {api_key}" - } + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"} logging.info(f"Sending request to {url}") try: @@ -43,20 +39,19 @@ def openai_chat_completions_request(url, api_key, data): except requests.exceptions.HTTPError as http_err: # Handle HTTP errors (e.g., response 4XX, 5XX) logging.error(f"Got HTTPError, exception={http_err}, payload={data}") - raise http_err + raise except requests.exceptions.RequestException as req_err: # Handle other requests-related errors (e.g., connection error) logging.warning(f"Got RequestException, exception={req_err}") - raise req_err + raise except Exception as e: # Handle other potential errors logging.warning(f"Got unknown Exception, exception={e}") - raise e + raise def ollama_chat_completions_request(messages, model): - """sends chat request to model running on Ollama""" - + """Sends chat request to model running on Ollama.""" url = "http://localhost:11434/api/chat" data = {"model": model, "messages": messages, "stream": False} @@ -71,12 +66,12 @@ def ollama_chat_completions_request(messages, model): except requests.exceptions.HTTPError as http_err: # Handle HTTP errors (e.g., response 4XX, 5XX) logging.error(f"Got HTTPError, exception={http_err}, payload={data}") - raise http_err + raise except requests.exceptions.RequestException as req_err: # Handle other requests-related errors (e.g., connection error) logging.warning(f"Got RequestException, exception={req_err}") - raise req_err + raise except Exception as e: # Handle other potential errors logging.warning(f"Got unknown Exception, exception={e}") - raise e + raise diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py index fef2da553dee3..9d429cdbf887e 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py @@ -6,7 +6,6 @@ class BaseMemory(ABC): - def __init__(self, file_name: str, entity: str = None): """Initializes the memory storage.""" self.file_name = file_name @@ -16,24 +15,20 @@ def __init__(self, file_name: str, entity: str = None): self.init_memory() @abstractmethod - def __len__(self): + def __len__(self) -> int: """Returns the number of items in the memory.""" - pass @abstractmethod def init_memory(self): """Initializes memory.""" - pass @abstractmethod def load_memory_from_file(self): """Loads memory from a file.""" - pass @abstractmethod def add_memory(self, data): """Adds new memory data.""" - pass @abstractmethod def get_memory(self): @@ -46,18 +41,18 @@ def return_memory(self): def remove_old_memory(self, days): """Removes memory items older than a specified number of days.""" cutoff_date = datetime.now() - timedelta(days=days) - self.memory = [ - item for item in self.return_memory if item.date >= cutoff_date - ] + self.memory = [item for item in self.return_memory if item.date >= cutoff_date] logging.info("Old memory removed successfully.") def save_memory(self): if self.file_name: - with open(self.file_name, 'w') as file: - json.dump([item.to_dict() for item in self.return_memory], - file, - default=str, - indent=4) + with open(self.file_name, "w") as file: + json.dump( + [item.to_dict() for item in self.return_memory], + file, + default=str, + indent=4, + ) logging.info(f"Memory saved to {self.file_name} successfully.") else: logging.info("No file name provided. Memory not saved.") @@ -80,7 +75,7 @@ def remove_memory_by_index(self, index): def clear_memory(self): self.memory = [] if self.file_name: - with open(self.file_name, 'w') as file: + with open(self.file_name, "w") as file: json.dump([], file, indent=4) logging.info( f"Memory cleared and saved to {self.file_name} successfully." diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py index 691990ea34266..ae3d1c9926f65 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, asdict +from dataclasses import dataclass from datetime import datetime @@ -7,16 +7,15 @@ class MemoryItem: entity: str date: datetime - def __str__(self): + def __str__(self) -> str: return f"{self.entity}, {self.date.isoformat()}" def to_dict(self): - return {'entity': self.entity, 'date': str(self.date.isoformat())} + return {"entity": self.entity, "date": str(self.date.isoformat())} @classmethod def from_dict(cls, data): - return cls(entity=data['entity'], - date=datetime.fromisoformat(data['date'])) + return cls(entity=data["entity"], date=datetime.fromisoformat(data["date"])) @dataclass @@ -25,18 +24,20 @@ class KnowledgeMemoryItem: count: int date: datetime - def __str__(self): + def __str__(self) -> str: return f"{self.entity}, {self.count}, {self.date.isoformat()}" def to_dict(self): return { - 'entity': self.entity, - 'count': self.count, - 'date': str(self.date.isoformat()) + "entity": self.entity, + "count": self.count, + "date": str(self.date.isoformat()), } @classmethod def from_dict(cls, data): - return cls(entity=data['entity'], - count=data['count'], - date=datetime.fromisoformat(data['date'])) + return cls( + entity=data["entity"], + count=data["count"], + date=datetime.fromisoformat(data["date"]), + ) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py index 43ad1af78a1b3..20b1185538936 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py @@ -6,13 +6,13 @@ class EntityKnowledgeStore(BaseMemory): - - def __len__(self): + def __len__(self) -> int: """Returns the number of items in the memory.""" return len(self.knowledge_memory) def init_memory(self): """Initializes memory. + self.entity_memory: list[EntityMemoryItem] """ self.load_memory_from_file() @@ -25,10 +25,9 @@ def return_memory(self): def load_memory_from_file(self): try: - with open(self.file_name, 'r') as file: + with open(self.file_name) as file: self.knowledge_memory = [ - KnowledgeMemoryItem.from_dict(item) - for item in json.load(file) + KnowledgeMemoryItem.from_dict(item) for item in json.load(file) ] logging.info( f"Entity Knowledge Memory loaded from {self.file_name} successfully." @@ -39,18 +38,18 @@ def load_memory_from_file(self): ) def add_memory(self, memory_stream: list[MemoryItem]): - """To add new memory to the entity knowledge store - we should convert the memory to knowledge memory and then update the knowledge memory + """Add new memory to the entity knowledge store. + + We should convert the memory to knowledge memory and then update the knowledge memory. Args: memory_stream (list): list of MemoryItem """ - knowledge_meory = self._convert_memory_to_knowledge_memory( - memory_stream) + knowledge_meory = self._convert_memory_to_knowledge_memory(memory_stream) self._update_knowledge_memory(knowledge_meory) def _update_knowledge_memory(self, knowledge_memory: list): - """update self.knowledge memory with new knowledge memory items + """Update self.knowledge memory with new knowledge memory items. Args: knowledge_memory (list): list of KnowledgeMemoryItem @@ -65,22 +64,23 @@ def _update_knowledge_memory(self, knowledge_memory: list): self.knowledge_memory.append(item) def _convert_memory_to_knowledge_memory( - self, memory_stream: list) -> list[KnowledgeMemoryItem]: - """Converts memory to knowledge memory + self, memory_stream: list + ) -> list[KnowledgeMemoryItem]: + """Converts memory to knowledge memory. Returns: knowledge_memory (list): list of KnowledgeMemoryItem """ knowledge_memory = [] - entities = set([item.entity for item in memory_stream]) + entities = {item.entity for item in memory_stream} for entity in entities: memory_dates = [ item.date for item in memory_stream if item.entity == entity ] knowledge_memory.append( - KnowledgeMemoryItem(entity, len(memory_dates), - max(memory_dates))) + KnowledgeMemoryItem(entity, len(memory_dates), max(memory_dates)) + ) return knowledge_memory def get_memory(self) -> list[KnowledgeMemoryItem]: diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py index f925ad3d4227c..529e7ee4a0215 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py @@ -9,13 +9,13 @@ class MemoryStream(BaseMemory): - - def __len__(self): + def __len__(self) -> int: """Returns the number of items in the memory.""" return len(self.memory) def init_memory(self): - """Initializes memory + """Initializes memory. + self.memory: list[MemoryItem] """ self.load_memory_from_file() @@ -27,21 +27,20 @@ def return_memory(self): return self.memory def add_memory(self, entities): - self.memory.extend([ - MemoryItem(str(entity), - datetime.now().replace(microsecond=0)) - for entity in entities - ]) + self.memory.extend( + [ + MemoryItem(str(entity), datetime.now().replace(microsecond=0)) + for entity in entities + ] + ) def get_memory(self) -> list[MemoryItem]: return self.memory def load_memory_from_file(self): try: - with open(self.file_name, 'r') as file: - self.memory = [ - MemoryItem.from_dict(item) for item in json.load(file) - ] + with open(self.file_name) as file: + self.memory = [MemoryItem.from_dict(item) for item in json.load(file)] logging.info(f"Memory loaded from {self.file_name} successfully.") except FileNotFoundError: logging.info("File not found. Starting with an empty memory.") diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py index 9c3dd1e6ff88e..a02b97dc93719 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py @@ -1,4 +1,4 @@ from llama_index.packs.memary.synonym_expand.synonym import custom_synonym_expand_fn from llama_index.packs.memary.synonym_expand.output import SynonymOutput -__all__ = ["custom_synonym_expand_fn", "SynonymOutput"] \ No newline at end of file +__all__ = ["custom_synonym_expand_fn", "SynonymOutput"] diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py index 56818a1198c00..2b787eb24820e 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py @@ -5,6 +5,5 @@ class SynonymOutput(BaseModel): synonyms: List[str] = Field( - description= - "Synonyms of keywords provided, make every letter lowercase except for the first letter" + description="Synonyms of keywords provided, make every letter lowercase except for the first letter" ) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py index 2e7c19f257647..e9e46c289d7c5 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py @@ -13,14 +13,14 @@ def custom_synonym_expand_fn(keywords: str) -> List[str]: parser = JsonOutputParser(pydantic_object=SynonymOutput) template = """ - You are an expert synonym exapnding system. Find synonyms or words commonly used in place to reference the same word for every word in the list: + You are an expert synonym expanding system. Find synonyms or words commonly used in place to reference the same word for every word in the list: Some examples are: - a synonym for Palantir may be Palantir technologies or Palantir technologies inc. - a synonym for Austin may be Austin texas - - a synonym for Taylor swift may be Taylor + - a synonym for Taylor swift may be Taylor - a synonym for Winter park may be Winter park resort - + Format: {format_instructions} Text: {keywords} @@ -29,20 +29,18 @@ def custom_synonym_expand_fn(keywords: str) -> List[str]: prompt = PromptTemplate( template=template, input_variables=["keywords"], - partial_variables={ - "format_instructions": parser.get_format_instructions() - }, + partial_variables={"format_instructions": parser.get_format_instructions()}, ) chain = prompt | llm | parser result = chain.invoke({"keywords": keywords}) - l = [] + synonyms_list = [] for category in result: for synonym in result[category]: - l.append(synonym.capitalize()) + synonyms_list.append(synonym.capitalize()) - return l + return synonyms_list # testing diff --git a/llama-index-packs/llama-index-packs-memary/pyproject.toml b/llama-index-packs/llama-index-packs-memary/pyproject.toml index 58188acc07884..0632b36010f3c 100644 --- a/llama-index-packs/llama-index-packs-memary/pyproject.toml +++ b/llama-index-packs/llama-index-packs-memary/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" +requires = ["poetry-core"] [tool.codespell] check-filenames = true @@ -24,43 +24,43 @@ ignore_missing_imports = true python_version = "3.10.12" [tool.poetry] -name = "llama-index-packs-memary" -version = "0.1.3" -description = "llama-index packs memary integration" authors = ["Seyeong Han "] +description = "llama-index packs memary integration" license = "MIT" -readme = "README.md" +name = "llama-index-packs-memary" packages = [{include = "llama_index/"}] +readme = "README.md" +version = "0.1.3" [tool.poetry.dependencies] python = ">3.9.7,<3.11.10" -neo4j="5.17.0" -python-dotenv="1.0.1" -pyvis="0.3.2" -streamlit="1.31.1" -llama-index="0.10.11" -llama-index-agent-openai="0.1.5" -llama-index-core="0.10.12" -llama-index-embeddings-openai="0.1.5" -llama-index-graph-stores-nebula="0.1.2" -llama-index-graph-stores-neo4j="0.1.1" -llama-index-legacy="0.9.48" -llama-index-llms-openai="0.1.5" -llama-index-multi-modal-llms-openai="0.1.3" -llama-index-program-openai="0.1.3" -llama-index-question-gen-openai="0.1.2" -llama-index-readers-file="0.1.4" -langchain="0.1.12" -langchain-openai="0.0.8" -llama-index-llms-perplexity="0.1.3" -llama-index-multi-modal-llms-ollama="0.1.3" -llama-index-llms-ollama="0.1.2" -pandas="2.2.0" -geocoder="1.38.1" -googlemaps="4.10.0" -ansistrip="0.1" -numpy="1.26.4" -ollama="0.1.9" +neo4j = "5.17.0" +python-dotenv = "1.0.1" +pyvis = "0.3.2" +streamlit = "1.31.1" +llama-index = "0.10.11" +llama-index-agent-openai = "0.1.5" +llama-index-core = "0.10.12" +llama-index-embeddings-openai = "0.1.5" +llama-index-graph-stores-nebula = "0.1.2" +llama-index-graph-stores-neo4j = "0.1.1" +llama-index-legacy = "0.9.48" +llama-index-llms-openai = "0.1.5" +llama-index-multi-modal-llms-openai = "0.1.3" +llama-index-program-openai = "0.1.3" +llama-index-question-gen-openai = "0.1.2" +llama-index-readers-file = "0.1.4" +langchain = "0.1.12" +langchain-openai = "0.0.8" +llama-index-llms-perplexity = "0.1.3" +llama-index-multi-modal-llms-ollama = "0.1.3" +llama-index-llms-ollama = "0.1.2" +pandas = "2.2.0" +geocoder = "1.38.1" +googlemaps = "4.10.0" +ansistrip = "0.1" +numpy = "1.26.4" +ollama = "0.1.9" [tool.poetry.group.dev.dependencies] black = {extras = ["jupyter"], version = "<=23.9.1,>=23.7.0"} @@ -78,5 +78,5 @@ types-Deprecated = ">=0.1.0" types-PyYAML = "^6.0.12.12" types-protobuf = "^4.24.0.4" types-redis = "4.5.5.0" -types-requests = "2.28.11.8" # TODO: unpin when mypy>0.991 +types-requests = "2.28.11.8" # TODO: unpin when mypy>0.991 types-setuptools = "67.1.0.0" diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json b/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json index 0637a088a01e8..fe51488c7066f 100644 --- a/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json +++ b/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json @@ -1 +1 @@ -[] \ No newline at end of file +[] diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json b/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json index 0637a088a01e8..fe51488c7066f 100644 --- a/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json +++ b/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json @@ -1 +1 @@ -[] \ No newline at end of file +[] diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json b/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json index 0637a088a01e8..fe51488c7066f 100644 --- a/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json +++ b/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json @@ -1 +1 @@ -[] \ No newline at end of file +[] diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt b/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt index bb28692261a77..3e95665b4d3e3 100644 --- a/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt +++ b/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt @@ -1,5 +1,5 @@ You are tasked with acting as a sophisticated conversational agent, uniquely equipped with a dual-layered memory system. This system consists of a Memory Stream and an Entity Knowledge Store. The Memory Stream captures all entities involved in conversations which are stored in knowledge graphs to infer users’ breadth of knowledge - ranging from questions and answers to their timestamps. Simultaneously, the Entity Knowledge Store captures the frequency and recency of the entities stored in the Memory Stream to infer users’ depth of knowledge. -Your primary role is to deliver a personalized and context-relevant final response that is two sentences or less by using the most recently added user query and ReAct agent response as primary information, and combining that with other information accessible via recent interactions and the top entities stored in the Entity Knowledge Store. You must comprehend the user's persona, incorporating their experiences, preferences, and personal details into your knowledge base outputting the final response. +Your primary role is to deliver a personalized and context-relevant final response that is two sentences or less by using the most recently added user query and ReAct agent response as primary information, and combining that with other information accessible via recent interactions and the top entities stored in the Entity Knowledge Store. You must comprehend the user's persona, incorporating their experiences, preferences, and personal details into your knowledge base outputting the final response. The final response should not include ReAct agent information outside of the most recently added one unless relevant to the most recently asked question. You are to interpret and apply the following data structures for personalized responses. Use the latest ReAct agent entry, Entity Knowledge Store, and Contexts to provide a final response to the latest question: @@ -8,4 +8,4 @@ ReAct agent: Answer generated by the routing agent using its tools. Represents t Entity Knowledge Store: A record of entities, including a count reflecting their frequency and the date of the last time the entity was inserted into the Memory Stream. If any of these entities, especially those with high counts, are included in the agent response, don't elaborate or explain the concept - you can infer the user is well familiar with the concepts. Do not repeat the Entity Knowledge Store back in the response. Interaction Keys: 'user' for user questions and 'system' for system-generated answers -Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user’s questions and the overarching conversation context. When addressing the user’s latest inquiry, your answer must integrate the current conversation's context, historical interactions, the latest response from the ReAct agent as well as the top chosen entities from the entity knowledge store. Keep final responses to two sentences. \ No newline at end of file +Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user’s questions and the overarching conversation context. When addressing the user’s latest inquiry, your answer must integrate the current conversation's context, historical interactions, the latest response from the ReAct agent as well as the top chosen entities from the entity knowledge store. Keep final responses to two sentences. diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt index d4c9b144e07e5..605306b6ec45c 100644 --- a/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt +++ b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt @@ -6,4 +6,4 @@ I really love sports such as badminton, tennis and workout and spend 4 or 5 day [Answer Instructions] I hope you can remember my questions and their answers so that I can leverage my personal past knowledge from you to be helpful in my life. -I always prefer short and concise answers, not over two sentences. \ No newline at end of file +I always prefer short and concise answers, not over two sentences. diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt index 1c72b94203486..5feed32b5c057 100644 --- a/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt +++ b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt @@ -6,4 +6,4 @@ I have worked as [insert experience, field involved in, length of experience and [Answer Instructions] Remember my past questions and their answers so that you can leverage my past knowledge and experiences to assist me in future interactions. -I always prefer short and concise answers, not over two sentences. \ No newline at end of file +I always prefer short and concise answers, not over two sentences. diff --git a/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py b/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py index 9f7b3f09a2991..15f0366fb6d96 100644 --- a/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py +++ b/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py @@ -1,16 +1,14 @@ import os import unittest -from datetime import datetime, timedelta -from llama_index.packs.memary import ChatAgent, EntityKnowledgeStore, MemoryStream +from datetime import datetime +from llama_index.packs.memary import EntityKnowledgeStore, MemoryStream from llama_index.packs.memary.memory.data_types import KnowledgeMemoryItem, MemoryItem class TestEntityKnowledgeStore(unittest.TestCase): - def setUp(self): self.file_name = "tests/memory/test_knowledge_memory.json" - self.entity_knowledge_store = EntityKnowledgeStore( - file_name=self.file_name) + self.entity_knowledge_store = EntityKnowledgeStore(file_name=self.file_name) def tearDown(self): # Clean up test file after each test @@ -20,29 +18,26 @@ def tearDown(self): pass def test_add_memory(self): - data = [ - MemoryItem("test_entity", - datetime.now().replace(microsecond=0)) - ] + data = [MemoryItem("test_entity", datetime.now().replace(microsecond=0))] self.entity_knowledge_store.add_memory(data) assert len(self.entity_knowledge_store.knowledge_memory) == 1 - assert isinstance(self.entity_knowledge_store.knowledge_memory[0], - KnowledgeMemoryItem) + assert isinstance( + self.entity_knowledge_store.knowledge_memory[0], KnowledgeMemoryItem + ) def test_convert_memory_to_knowledge_memory(self): - data = [ - MemoryItem("test_entity", - datetime.now().replace(microsecond=0)) - ] - converted_data = self.entity_knowledge_store._convert_memory_to_knowledge_memory( - data) + data = [MemoryItem("test_entity", datetime.now().replace(microsecond=0))] + converted_data = ( + self.entity_knowledge_store._convert_memory_to_knowledge_memory(data) + ) assert len(converted_data) == 1 assert isinstance(converted_data[0], KnowledgeMemoryItem) def test_update_knowledge_memory(self): data = [ - KnowledgeMemoryItem("knowledge_entity", 1, - datetime.now().replace(microsecond=0)) + KnowledgeMemoryItem( + "knowledge_entity", 1, datetime.now().replace(microsecond=0) + ) ] self.entity_knowledge_store._update_knowledge_memory(data) assert len(self.entity_knowledge_store.knowledge_memory) == 1 @@ -50,7 +45,6 @@ def test_update_knowledge_memory(self): class TestMemoryStream(unittest.TestCase): - def setUp(self): self.file_name = "tests/test_memory.json" self.memory_stream = MemoryStream(file_name=self.file_name) @@ -63,13 +57,11 @@ def tearDown(self): pass def test_save_and_load_memory(self): - data = [ - MemoryItem("test_entity", - datetime.now().replace(microsecond=0)) - ] + data = [MemoryItem("test_entity", datetime.now().replace(microsecond=0))] self.memory_stream.add_memory(data) self.memory_stream.save_memory() new_memory_stream = MemoryStream(file_name=self.file_name) self.assertEqual(len(new_memory_stream), len(self.memory_stream)) - self.assertEqual(new_memory_stream.get_memory(), - self.memory_stream.get_memory()) + self.assertEqual( + new_memory_stream.get_memory(), self.memory_stream.get_memory() + ) From f5d3b4c34b79d47035d941930fa72e0faa65cf4a Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Thu, 6 Jun 2024 23:12:05 -0500 Subject: [PATCH 16/24] fix: add target sources --- llama-index-packs/llama-index-packs-memary/BUILD | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llama-index-packs/llama-index-packs-memary/BUILD b/llama-index-packs/llama-index-packs-memary/BUILD index db46e8d6c978c..9ad593826d30a 100644 --- a/llama-index-packs/llama-index-packs-memary/BUILD +++ b/llama-index-packs/llama-index-packs-memary/BUILD @@ -1 +1,3 @@ -python_sources() +python_sources( + sources=["llama_index/packs/memary/*.py"], +) From 495ea8a67f082aa8f7f69db41b989bfaa658a978 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Wed, 12 Jun 2024 10:26:51 -0500 Subject: [PATCH 17/24] fix: add poetry_requirements after running `pants tailor` --- llama-index-packs/llama-index-packs-memary/BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llama-index-packs/llama-index-packs-memary/BUILD b/llama-index-packs/llama-index-packs-memary/BUILD index 9ad593826d30a..446fdcb4b4fad 100644 --- a/llama-index-packs/llama-index-packs-memary/BUILD +++ b/llama-index-packs/llama-index-packs-memary/BUILD @@ -1,3 +1,7 @@ python_sources( sources=["llama_index/packs/memary/*.py"], ) + +poetry_requirements( + name="poetry", +) From aefa471b9fddb3bbe351e55fd92a9fe22ed5828c Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Sun, 16 Jun 2024 22:59:43 -0500 Subject: [PATCH 18/24] refactor: remove and replace with memary library --- .../packs/memary/agent/__init__.py | 18 - .../packs/memary/agent/base_agent.py | 490 ------------------ .../packs/memary/agent/chat_agent.py | 80 --- .../packs/memary/agent/data_types.py | 98 ---- .../packs/memary/agent/llm_api/tools.py | 77 --- .../packs/memary/memory/__init__.py | 5 - .../packs/memary/memory/base_memory.py | 84 --- .../packs/memary/memory/data_types.py | 43 -- .../memary/memory/entity_knowledge_store.py | 87 ---- .../packs/memary/memory/memory_stream.py | 46 -- .../packs/memary/synonym_expand/__init__.py | 4 - .../packs/memary/synonym_expand/output.py | 9 - .../packs/memary/synonym_expand/synonym.py | 47 -- 13 files changed, 1088 deletions(-) delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/__init__.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py delete mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py deleted file mode 100644 index 22fefcbc2e144..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from llama_index.packs.memary.agent.base_agent import Agent -from llama_index.packs.memary.agent.chat_agent import ChatAgent -from llama_index.packs.memary.agent.llm_api.tools import ( - ollama_chat_completions_request, - openai_chat_completions_request, -) -from llama_index.packs.memary.memory import EntityKnowledgeStore, MemoryStream -from llama_index.packs.memary.synonym_expand.synonym import custom_synonym_expand_fn - -__all__ = [ - "Agent", - "ChatAgent", - "EntityKnowledgeStore", - "MemoryStream", - "ollama_chat_completions_request", - "openai_chat_completions_request", - "custom_synonym_expand_fn", -] diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py deleted file mode 100644 index 5bd1a91ace53b..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/base_agent.py +++ /dev/null @@ -1,490 +0,0 @@ -import logging -import os -import sys -from pathlib import Path -from typing import Any, Callable, Dict, List - -import geocoder -import googlemaps -import numpy as np -import requests -from ansistrip import ansi_strip -from dotenv import load_dotenv -from llama_index.core import ( - KnowledgeGraphIndex, - Settings, - SimpleDirectoryReader, - StorageContext, -) -from llama_index.core.agent import ReActAgent -from llama_index.core.llms import ChatMessage -from llama_index.core.query_engine import RetrieverQueryEngine -from llama_index.core.retrievers import KnowledgeGraphRAGRetriever -from llama_index.core.tools import FunctionTool -from llama_index.graph_stores.neo4j import Neo4jGraphStore -from llama_index.llms.ollama import Ollama -from llama_index.llms.openai import OpenAI -from llama_index.llms.perplexity import Perplexity -from llama_index.multi_modal_llms.ollama import OllamaMultiModal -from llama_index.multi_modal_llms.openai import OpenAIMultiModal - -from llama_index.packs.memary.agent.data_types import Context, Message -from llama_index.packs.memary.agent.llm_api.tools import ( - ollama_chat_completions_request, - openai_chat_completions_request, -) -from llama_index.packs.memary.memory import EntityKnowledgeStore, MemoryStream -from llama_index.packs.memary.synonym_expand import custom_synonym_expand_fn - -MAX_ENTITIES_FROM_KG = 5 -ENTITY_EXCEPTIONS = ["Unknown relation"] -# LLM token limits -CONTEXT_LENGTH = 4096 -EVICTION_RATE = 0.7 -NONEVICTION_LENGTH = 5 -TOP_ENTITIES = 20 - - -def generate_string(entities): - cypher_query = "MATCH p = (n) - [*1 .. 2] - ()\n" - cypher_query += "WHERE n.id IN " + str(entities) + "\n" - cypher_query += "RETURN p" - - return cypher_query - - -class Agent: - """Agent manages the RAG model, the ReAct agent, and the memory stream.""" - - def __init__( - self, - name, - memory_stream_json, - entity_knowledge_store_json, - system_persona_txt, - user_persona_txt, - past_chat_json, - llm_model_name="llama3", - vision_model_name="llava", - include_from_defaults=["search", "locate", "vision", "stocks"], - debug=True, - ) -> None: - load_dotenv() - self.name = name - self.model = llm_model_name - - googlemaps_api_key = os.getenv("GOOGLEMAPS_API_KEY") - pplx_api_key = os.getenv("PERPLEXITY_API_KEY") - - # Neo4j credentials - self.neo4j_username = "neo4j" - self.neo4j_password = os.getenv("NEO4J_PW") - self.neo4j_url = os.getenv("NEO4J_URL") - database = "neo4j" - - # initialize APIs - self.load_llm_model(llm_model_name) - self.load_vision_model(vision_model_name) - self.query_llm = Perplexity( - api_key=pplx_api_key, model="mistral-7b-instruct", temperature=0.5 - ) - self.gmaps = googlemaps.Client(key=googlemaps_api_key) - Settings.llm = self.llm - Settings.chunk_size = 512 - - # initialize Neo4j graph resources - self.graph_store = Neo4jGraphStore( - username=self.neo4j_username, - password=self.neo4j_password, - url=self.neo4j_url, - database=database, - ) - - self.vantage_key = os.getenv("ALPHA_VANTAGE_API_KEY") - - self.storage_context = StorageContext.from_defaults( - graph_store=self.graph_store - ) - graph_rag_retriever = KnowledgeGraphRAGRetriever( - storage_context=self.storage_context, - verbose=True, - llm=self.llm, - retriever_mode="keyword", - synonym_expand_fn=custom_synonym_expand_fn, - ) - - self.query_engine = RetrieverQueryEngine.from_args(graph_rag_retriever) - - self.debug = debug - self.tools = {} - self._init_default_tools(default_tools=include_from_defaults) - - self.memory_stream = MemoryStream(memory_stream_json) - self.entity_knowledge_store = EntityKnowledgeStore(entity_knowledge_store_json) - - self.message = Message( - system_persona_txt, user_persona_txt, past_chat_json, self.model - ) - - def __str__(self) -> str: - return f"Agent {self.name}" - - def load_llm_model(self, llm_model_name): - if llm_model_name == "gpt-3.5-turbo": - os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY") - self.openai_api_key = os.environ["OPENAI_API_KEY"] - self.model_endpoint = "https://api.openai.com/v1" - self.llm = OpenAI(model="gpt-3.5-turbo-instruct") - else: - try: - self.llm = Ollama(model=llm_model_name, request_timeout=60.0) - except Exception: - raise ValueError("Please provide a proper llm_model_name.") - - def load_vision_model(self, vision_model_name): - if vision_model_name == "gpt-4-vision-preview": - os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY") - self.openai_api_key = os.environ["OPENAI_API_KEY"] - self.mm_model = OpenAIMultiModal( - model="gpt-4-vision-preview", - api_key=os.getenv("OPENAI_KEY"), - max_new_tokens=300, - ) - else: - try: - self.mm_model = OllamaMultiModal(model=vision_model_name) - except Exception: - raise ValueError("Please provide a proper vision_model_name.") - - def external_query(self, query: str): - messages_dict = [ - {"role": "system", "content": "Be precise and concise."}, - {"role": "user", "content": query}, - ] - messages = [ChatMessage(**msg) for msg in messages_dict] - external_response = self.query_llm.chat(messages) - - return str(external_response) - - def search(self, query: str) -> str: - """Search the knowledge graph or perform search on the web if information is not present in the knowledge graph.""" - response = self.query_engine.query(query) - - if response.metadata is None: - return self.external_query(query) - else: - return response - - def locate(self, query: str) -> str: - """Finds the current geographical location.""" - location = geocoder.ip("me") - latitude, longitude = location.latlng[0], location.latlng[1] - - reverse_geocode_result = self.gmaps.reverse_geocode((latitude, longitude)) - formatted_address = reverse_geocode_result[0]["formatted_address"] - return "Your address is" + formatted_address - - def vision(self, query: str, img_url: str) -> str: - """Uses computer vision to process the image specified by the image url and answers the question based on the CV results.""" - query_image_dir_path = Path("query_images") - if not query_image_dir_path.exists(): - Path.mkdir(query_image_dir_path) - - data = requests.get(img_url).content - query_image_path = os.path.join(query_image_dir_path, "query.jpg") - with open(query_image_path, "wb") as f: - f.write(data) - image_documents = SimpleDirectoryReader(query_image_dir_path).load_data() - - response = self.mm_model.complete(prompt=query, image_documents=image_documents) - - os.remove(query_image_path) # delete image after use - return response - - def stocks(self, query: str) -> str: - """Get the stock price of the company given the ticker.""" - request_api = requests.get( - r"https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol=" - + query - + r"&apikey=" - + self.vantage_key - ) - return request_api.json() - - # def get_news(self, query: str) -> str: - # """Given a keyword, search for news articles related to the keyword""" - # request_api = requests.get(r'https://newsdata.io/api/1/news?apikey=' + self.news_data_key + r'&q=' + query) - # return request_api.json() - - def query(self, query: str) -> str: - # get the response from react agent - response = self.routing_agent.chat(query) - self.routing_agent.reset() - # write response to file for KG writeback - with open("data/external_response.txt", "w") as f: - print(response, file=f) - # write back to the KG - self.write_back() - return response - - def write_back(self): - documents = SimpleDirectoryReader( - input_files=["data/external_response.txt"] - ).load_data() - - KnowledgeGraphIndex.from_documents( - documents, - storage_context=self.storage_context, - max_triplets_per_chunk=8, - ) - - def check_KG(self, query: str) -> bool: - """Check if the query is in the knowledge graph. - - Args: - query (str): query to check in the knowledge graph - - Returns: - bool: True if the query is in the knowledge graph, False otherwise - """ - response = self.query_engine.query(query) - - if response.metadata is None: - return False - return generate_string( - list(next(iter(response.metadata.values()))["kg_rel_map"].keys()) - ) - - def _select_top_entities(self): - entity_knowledge_store = self.message.llm_message["knowledge_entity_store"] - entities = [entity.to_dict() for entity in entity_knowledge_store] - entity_counts = [entity["count"] for entity in entities] - top_indexes = np.argsort(entity_counts)[:TOP_ENTITIES] - return [entities[index] for index in top_indexes] - - def _add_contexts_to_llm_message(self, role, content, index=None): - """Add contexts to the llm_message.""" - if index: - self.message.llm_message["messages"].insert(index, Context(role, content)) - else: - self.message.llm_message["messages"].append(Context(role, content)) - - def _change_llm_message_chat(self) -> dict: - """Change the llm_message to chat format. - - Returns: - dict: llm_message in chat format - """ - llm_message_chat = self.message.llm_message.copy() - llm_message_chat["messages"] = [] - top_entities = self._select_top_entities() - logging.info(f"top_entities: {top_entities}") - llm_message_chat["messages"].append( - { - "role": "user", - "content": "Knowledge Entity Store:" + str(top_entities), - } - ) - llm_message_chat["messages"].extend( - [context.to_dict() for context in self.message.llm_message["messages"]] - ) - llm_message_chat.pop("knowledge_entity_store") - llm_message_chat.pop("memory_stream") - return llm_message_chat - - def _summarize_contexts(self, total_tokens: int): - """Summarize the contexts. - - Args: - total_tokens (int): total tokens in the response - """ - messages = self.message.llm_message["messages"] - - # First two messages are system and user personas - if len(messages) > 2 + NONEVICTION_LENGTH: - messages = messages[2:-NONEVICTION_LENGTH] - del self.message.llm_message["messages"][2:-NONEVICTION_LENGTH] - else: - messages = messages[2:] - del self.message.llm_message["messages"][2:] - - message_contents = [message.to_dict()["content"] for message in messages] - - llm_message_chat = { - "model": self.model, - "messages": [ - { - "role": "user", - "content": "Summarize these previous conversations into 50 words:" - + str(message_contents), - } - ], - } - response, _ = self._get_chat_response(llm_message_chat) - content = "Summarized past conversation:" + response - self._add_contexts_to_llm_message("assistant", content, index=2) - logging.info(f"Contexts summarized successfully. \n summary: {response}") - logging.info(f"Total tokens after eviction: {total_tokens*EVICTION_RATE}") - - def _get_chat_response(self, llm_message_chat: str) -> str: - """Get response from the LLM chat model. - - Args: - llm_message_chat (str): query to get response for - - Returns: - str: response from the LLM chat model - """ - if self.model == "gpt-3.5-turbo": - response = openai_chat_completions_request( - self.model_endpoint, self.openai_api_key, llm_message_chat - ) - total_tokens = response["usage"]["total_tokens"] - response = str(response["choices"][0]["message"]["content"]) - else: # default to Ollama model - response = ollama_chat_completions_request( - llm_message_chat["messages"], self.model - ) - total_tokens = response.get( - "prompt_eval_count", 0 - ) # if 'prompt_eval_count' not present then query is cached - response = str(response["message"]["content"]) - return response, total_tokens - - def get_response(self) -> str: - """Get response from the RAG model. - - Returns: - str: response from the RAG model - """ - llm_message_chat = self._change_llm_message_chat() - response, total_tokens = self._get_chat_response(llm_message_chat) - if total_tokens > CONTEXT_LENGTH * EVICTION_RATE: - logging.info("Evicting and summarizing contexts") - self._summarize_contexts(total_tokens) - - self.message.save_contexts_to_json() - - return response - - def get_routing_agent_response(self, query, return_entity=False): - """Get response from the ReAct.""" - response = "" - if self.debug: - # writes ReAct agent steps to separate file and modifies format to be readable in .txt file - with open("data/routing_response.txt", "w") as f: - orig_stdout = sys.stdout - sys.stdout = f - response = str(self.query(query)) - sys.stdout.flush() - sys.stdout = orig_stdout - text = "" - with open("data/routing_response.txt") as f: - text = f.read() - - plain = ansi_strip(text) - with open("data/routing_response.txt", "w") as f: - f.write(plain) - else: - response = str(self.query(query)) - - if return_entity: - # the query above already adds final response to KG so entities will be present in the KG - return response, self.get_entity(self.query_engine.retrieve(query)) - return response - - def get_entity(self, retrieve) -> list[str]: - """Retrieve is a list of QueryBundle objects. - A retrieved QueryBundle object has a "node" attribute, - which has a "metadata" attribute. - - example for "kg_rel_map": - kg_rel_map = { - 'Harry': [['DREAMED_OF', 'Unknown relation'], ['FELL_HARD_ON', 'Concrete floor']], - 'Potter': [['WORE', 'Round glasses'], ['HAD', 'Dream']] - } - - Args: - retrieve (list[NodeWithScore]): list of NodeWithScore objects - return: - list[str]: list of string entities - """ - entities = [] - kg_rel_map = retrieve[0].node.metadata["kg_rel_map"] - for key, items in kg_rel_map.items(): - # key is the entity of question - entities.append(key) - # items is a list of [relationship, entity] - entities.extend(item[1] for item in items) - if len(entities) > MAX_ENTITIES_FROM_KG: - break - entities = list(set(entities)) - for exceptions in ENTITY_EXCEPTIONS: - if exceptions in entities: - entities.remove(exceptions) - return entities - - def _init_ReAct_agent(self): - """Initializes ReAct Agent with list of tools in self.tools.""" - tool_fns = [] - for func in self.tools.values(): - tool_fns.append(FunctionTool.from_defaults(fn=func)) - self.routing_agent = ReActAgent.from_tools(tool_fns, llm=self.llm, verbose=True) - - def _init_default_tools(self, default_tools: List[str]): - """Initializes ReAct Agent from the default list of tools memary provides. - List of strings passed in during initialization denoting which default tools to include. - - Args: - default_tools (list(str)): list of tool names in string form - """ - for tool in default_tools: - if tool == "search": - self.tools["search"] = self.search - elif tool == "locate": - self.tools["locate"] = self.locate - elif tool == "vision": - self.tools["vision"] = self.vision - elif tool == "stocks": - self.tools["stocks"] = self.stocks - self._init_ReAct_agent() - - def add_tool(self, tool_additions: Dict[str, Callable[..., Any]]): - """Adds specified tools to be used by the ReAct Agent. - - Args: - tools (dict(str, func)): dictionary of tools with names as keys and associated functions as values - """ - for tool_name in tool_additions: - self.tools[tool_name] = tool_additions[tool_name] - self._init_ReAct_agent() - - def remove_tool(self, tool_name: str): - """Removes specified tool from list of available tools for use by the ReAct Agent. - - Args: - tool_name (str): name of tool to be removed in string form - """ - if tool_name in self.tools: - del self.tools[tool_name] - self._init_ReAct_agent() - else: - raise ("Unknown tool_name provided for removal.") - - def update_tools(self, updated_tools: List[str]): - """Resets ReAct Agent tools to only include subset of default tools. - - Args: - updated_tools (list(str)): list of default tools to include - """ - self.tools.clear() - for tool in updated_tools: - if tool == "search": - self.tools["search"] = self.search - elif tool == "locate": - self.tools["locate"] = self.locate - elif tool == "vision": - self.tools["vision"] = self.vision - elif tool == "stocks": - self.tools["stocks"] = self.stocks - self._init_ReAct_agent() diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py deleted file mode 100644 index 75238d1698cdf..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/chat_agent.py +++ /dev/null @@ -1,80 +0,0 @@ -from typing import Optional, List - -from llama_index.packs.memary.agent.base_agent import Agent -import logging - - -class ChatAgent(Agent): - """ChatAgent currently able to support Llama3 running on Ollama (default) and gpt-3.5-turbo for llm models, - and LLaVA running on Ollama (default) and gpt-4-vision-preview for the vision tool. - """ - - def __init__( - self, - name, - memory_stream_json, - entity_knowledge_store_json, - system_persona_txt, - user_persona_txt, - past_chat_json, - llm_model_name="llama3", - vision_model_name="llava", - include_from_defaults=["search", "locate", "vision", "stocks"], - ) -> None: - super().__init__( - name, - memory_stream_json, - entity_knowledge_store_json, - system_persona_txt, - user_persona_txt, - past_chat_json, - llm_model_name, - vision_model_name, - include_from_defaults, - ) - - def add_chat(self, role: str, content: str, entities: Optional[List[str]] = None): - """Add a chat to the agent's memory. - - Args: - role (str): 'system' or 'user' - content (str): content of the chat - entities (Optional[List[str]], optional): entities from Memory systems. Defaults to None. - """ - # Add a chat to the agent's memory. - self._add_contexts_to_llm_message(role, content) - - if entities: - self.memory_stream.add_memory(entities) - self.memory_stream.save_memory() - self.entity_knowledge_store.add_memory(self.memory_stream.get_memory()) - self.entity_knowledge_store.save_memory() - - self._replace_memory_from_llm_message() - self._replace_eks_to_from_message() - - def get_chat(self): - return self.contexts - - def clearMemory(self): - """Clears Neo4j database and memory stream/entity knowledge store.""" - logging.info("Deleting memory stream and entity knowledge store...") - self.memory_stream.clear_memory() - self.entity_knowledge_store.clear_memory() - - logging.info("Deleting nodes from Neo4j...") - try: - self.graph_store.query("MATCH (n) DETACH DELETE n") - except Exception as e: - logging.error(f"Error deleting nodes: {e}") - logging.info("Nodes deleted from Neo4j.") - - def _replace_memory_from_llm_message(self): - """Replace the memory_stream from the llm_message.""" - self.message.llm_message["memory_stream"] = self.memory_stream.get_memory() - - def _replace_eks_to_from_message(self): - """Replace the entity knowledge store from the llm_message. eks means entity knowledge store.""" - self.message.llm_message[ - "knowledge_entity_store" - ] = self.entity_knowledge_store.get_memory() diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py deleted file mode 100644 index 2e87c65e2926a..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/data_types.py +++ /dev/null @@ -1,98 +0,0 @@ -import json -import logging -from dataclasses import dataclass - - -def save_json(filename, data): - with open(filename, "w") as f: - json.dump(data, f, indent=4) - - -@dataclass -class Context: - """Context class to store the role and content of the message.""" - - role: str # system or user - content: str - - def __str__(self) -> str: - return f"{self.role}: {self.content} |" - - def to_dict(self): - return {"role": self.role, "content": self.content} - - -class Message: - """Message class to store the contexts, memory stream and knowledge entity store.""" - - def __init__( - self, system_persona_txt, user_persona_txt, past_chat_json, model - ) -> None: - self.past_chat_json = past_chat_json - - self.contexts = [] - self.system_persona = self.load_persona(system_persona_txt) - self.user_persona = self.load_persona(user_persona_txt) - self._init_persona_to_messages() - self.contexts.extend(self.load_contexts_from_json()) - - self.llm_message = { - "model": model, - "messages": self.contexts, - "memory_stream": [], - "knowledge_entity_store": [], - } - - # self.prompt_tokens = count_tokens(self.contexts) - - def __str__(self) -> str: - llm_message_str = f"System Persona: {self.system_persona}\nUser Persona: {self.user_persona}\n" - for context in self.contexts: - llm_message_str += f"{context!s}," - for memory in self.llm_message["memory_stream"]: - llm_message_str += f"{memory!s}," - for entity in self.llm_message["knowledge_entity_store"]: - llm_message_str += f"{entity!s}," - return llm_message_str - - def _init_persona_to_messages(self): - """Initializes the system and user personas to the contexts.""" - self.contexts.append(Context("system", self.system_persona)) - self.contexts.append(Context("user", self.user_persona)) - - def load_persona(self, persona_txt) -> str: - """Loads the persona from the txt file. - - Args: - persona_txt (str): persona txt file path - - Returns: - str: persona - """ - try: - with open(persona_txt) as file: - return file.read() - except FileNotFoundError: - logging.info(f"{persona_txt} file does not exist.") - - def load_contexts_from_json(self): - """Loads the contexts from the past chat json file.""" - try: - with open(self.past_chat_json) as file: - data_dicts = json.load(file) - - return [Context(**data_dict) for data_dict in data_dicts] - except FileNotFoundError: - logging.info( - f"{self.past_chat_json} file does not exist. Starts from empty contexts." - ) - return [] - - def save_contexts_to_json(self): - """Saves the contexts to the json file. - We don't save the system and user personas (first two messages). - """ - save_json( - self.past_chat_json, - [message.to_dict() for message in self.llm_message["messages"][2:]], - ) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py deleted file mode 100644 index 38d76dbb63c2e..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/agent/llm_api/tools.py +++ /dev/null @@ -1,77 +0,0 @@ -# From MemGPT llm_api_tools.py: - -import urllib -import logging -import requests - - -def smart_urljoin(base_url, relative_url): - """Urljoin is stupid and wants a trailing / at the end of the endpoint address, or it will chop the suffix off.""" - if not base_url.endswith("/"): - base_url += "/" - return urllib.parse.urljoin(base_url, relative_url) - - -def openai_chat_completions_request(url, api_key, data): - """text-generation?lang=curl.""" - url = smart_urljoin(url, "chat/completions") - headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"} - - logging.info(f"Sending request to {url}") - try: - # Example code to trigger a rate limit response: - # mock_response = requests.Response() - # mock_response.status_code = 429 - # http_error = requests.exceptions.HTTPError("429 Client Error: Too Many Requests") - # http_error.response = mock_response - # raise http_error - - # Example code to trigger a context overflow response (for an 8k model) - # data["messages"][-1]["content"] = " ".join(["repeat after me this is not a fluke"] * 1000) - - response = requests.post(url, headers=headers, json=data) - logging.info(f"response = {response}") - response.raise_for_status() # Raises HTTPError for 4XX/5XX status - response = response.json() # convert to dict from string - logging.info(f"response.json = {response}") - # response = ChatCompletionResponse(**response) # convert to 'dot-dict' style which is the openai python client default - return response - except requests.exceptions.HTTPError as http_err: - # Handle HTTP errors (e.g., response 4XX, 5XX) - logging.error(f"Got HTTPError, exception={http_err}, payload={data}") - raise - except requests.exceptions.RequestException as req_err: - # Handle other requests-related errors (e.g., connection error) - logging.warning(f"Got RequestException, exception={req_err}") - raise - except Exception as e: - # Handle other potential errors - logging.warning(f"Got unknown Exception, exception={e}") - raise - - -def ollama_chat_completions_request(messages, model): - """Sends chat request to model running on Ollama.""" - url = "http://localhost:11434/api/chat" - data = {"model": model, "messages": messages, "stream": False} - - logging.info(f"Sending request to {url}") - try: - response = requests.post(url, json=data) - logging.info(f"response = {response}") - response.raise_for_status() - response = response.json() - logging.info(f"response.json = {response}") - return response - except requests.exceptions.HTTPError as http_err: - # Handle HTTP errors (e.g., response 4XX, 5XX) - logging.error(f"Got HTTPError, exception={http_err}, payload={data}") - raise - except requests.exceptions.RequestException as req_err: - # Handle other requests-related errors (e.g., connection error) - logging.warning(f"Got RequestException, exception={req_err}") - raise - except Exception as e: - # Handle other potential errors - logging.warning(f"Got unknown Exception, exception={e}") - raise diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/__init__.py deleted file mode 100644 index 9b9747d074f94..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from llama_index.packs.memary.memory.base_memory import BaseMemory -from llama_index.packs.memary.memory.memory_stream import MemoryStream -from llama_index.packs.memary.memory.entity_knowledge_store import EntityKnowledgeStore - -__all__ = ["BaseMemory", "MemoryStream", "EntityKnowledgeStore"] diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py deleted file mode 100644 index 9d429cdbf887e..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/base_memory.py +++ /dev/null @@ -1,84 +0,0 @@ -import json -import logging - -from abc import ABC, abstractmethod -from datetime import datetime, timedelta - - -class BaseMemory(ABC): - def __init__(self, file_name: str, entity: str = None): - """Initializes the memory storage.""" - self.file_name = file_name - self.entity = entity - self.memory = [] - self.knowledge_memory = [] - self.init_memory() - - @abstractmethod - def __len__(self) -> int: - """Returns the number of items in the memory.""" - - @abstractmethod - def init_memory(self): - """Initializes memory.""" - - @abstractmethod - def load_memory_from_file(self): - """Loads memory from a file.""" - - @abstractmethod - def add_memory(self, data): - """Adds new memory data.""" - - @abstractmethod - def get_memory(self): - pass - - @property - def return_memory(self): - return self.memory - - def remove_old_memory(self, days): - """Removes memory items older than a specified number of days.""" - cutoff_date = datetime.now() - timedelta(days=days) - self.memory = [item for item in self.return_memory if item.date >= cutoff_date] - logging.info("Old memory removed successfully.") - - def save_memory(self): - if self.file_name: - with open(self.file_name, "w") as file: - json.dump( - [item.to_dict() for item in self.return_memory], - file, - default=str, - indent=4, - ) - logging.info(f"Memory saved to {self.file_name} successfully.") - else: - logging.info("No file name provided. Memory not saved.") - - def get_memory_by_index(self, index): - if 0 <= index < len(self.memory): - return self.memory[index] - else: - return None - - def remove_memory_by_index(self, index): - if 0 <= index < len(self.memory): - del self.memory[index] - logging.info("Memory item removed successfully.") - return True - else: - logging.info("Invalid index. Memory item not removed.") - return False - - def clear_memory(self): - self.memory = [] - if self.file_name: - with open(self.file_name, "w") as file: - json.dump([], file, indent=4) - logging.info( - f"Memory cleared and saved to {self.file_name} successfully." - ) - else: - logging.info("No file name provided. Memory not cleared or saved.") diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py deleted file mode 100644 index ae3d1c9926f65..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/data_types.py +++ /dev/null @@ -1,43 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime - - -@dataclass -class MemoryItem: - entity: str - date: datetime - - def __str__(self) -> str: - return f"{self.entity}, {self.date.isoformat()}" - - def to_dict(self): - return {"entity": self.entity, "date": str(self.date.isoformat())} - - @classmethod - def from_dict(cls, data): - return cls(entity=data["entity"], date=datetime.fromisoformat(data["date"])) - - -@dataclass -class KnowledgeMemoryItem: - entity: str - count: int - date: datetime - - def __str__(self) -> str: - return f"{self.entity}, {self.count}, {self.date.isoformat()}" - - def to_dict(self): - return { - "entity": self.entity, - "count": self.count, - "date": str(self.date.isoformat()), - } - - @classmethod - def from_dict(cls, data): - return cls( - entity=data["entity"], - count=data["count"], - date=datetime.fromisoformat(data["date"]), - ) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py deleted file mode 100644 index 20b1185538936..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/entity_knowledge_store.py +++ /dev/null @@ -1,87 +0,0 @@ -import json -import logging - -from llama_index.packs.memary.memory import BaseMemory -from llama_index.packs.memary.memory.data_types import KnowledgeMemoryItem, MemoryItem - - -class EntityKnowledgeStore(BaseMemory): - def __len__(self) -> int: - """Returns the number of items in the memory.""" - return len(self.knowledge_memory) - - def init_memory(self): - """Initializes memory. - - self.entity_memory: list[EntityMemoryItem] - """ - self.load_memory_from_file() - if self.entity: - self.add_memory(self.entity) - - @property - def return_memory(self): - return self.knowledge_memory - - def load_memory_from_file(self): - try: - with open(self.file_name) as file: - self.knowledge_memory = [ - KnowledgeMemoryItem.from_dict(item) for item in json.load(file) - ] - logging.info( - f"Entity Knowledge Memory loaded from {self.file_name} successfully." - ) - except FileNotFoundError: - logging.info( - "File not found. Starting with an empty entity knowledge memory." - ) - - def add_memory(self, memory_stream: list[MemoryItem]): - """Add new memory to the entity knowledge store. - - We should convert the memory to knowledge memory and then update the knowledge memory. - - Args: - memory_stream (list): list of MemoryItem - """ - knowledge_meory = self._convert_memory_to_knowledge_memory(memory_stream) - self._update_knowledge_memory(knowledge_meory) - - def _update_knowledge_memory(self, knowledge_memory: list): - """Update self.knowledge memory with new knowledge memory items. - - Args: - knowledge_memory (list): list of KnowledgeMemoryItem - """ - for item in knowledge_memory: - for i, entity in enumerate(self.knowledge_memory): - if entity.entity == item.entity: - self.knowledge_memory[i].date = item.date - self.knowledge_memory[i].count += item.count - break - else: - self.knowledge_memory.append(item) - - def _convert_memory_to_knowledge_memory( - self, memory_stream: list - ) -> list[KnowledgeMemoryItem]: - """Converts memory to knowledge memory. - - Returns: - knowledge_memory (list): list of KnowledgeMemoryItem - """ - knowledge_memory = [] - - entities = {item.entity for item in memory_stream} - for entity in entities: - memory_dates = [ - item.date for item in memory_stream if item.entity == entity - ] - knowledge_memory.append( - KnowledgeMemoryItem(entity, len(memory_dates), max(memory_dates)) - ) - return knowledge_memory - - def get_memory(self) -> list[KnowledgeMemoryItem]: - return self.knowledge_memory diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py deleted file mode 100644 index 529e7ee4a0215..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/memory/memory_stream.py +++ /dev/null @@ -1,46 +0,0 @@ -import json -import logging -from datetime import datetime - -from llama_index.packs.memary.memory import BaseMemory -from llama_index.packs.memary.memory.data_types import MemoryItem - -logging.basicConfig(level=logging.INFO) - - -class MemoryStream(BaseMemory): - def __len__(self) -> int: - """Returns the number of items in the memory.""" - return len(self.memory) - - def init_memory(self): - """Initializes memory. - - self.memory: list[MemoryItem] - """ - self.load_memory_from_file() - if self.entity: - self.add_memory(self.entity) - - @property - def return_memory(self): - return self.memory - - def add_memory(self, entities): - self.memory.extend( - [ - MemoryItem(str(entity), datetime.now().replace(microsecond=0)) - for entity in entities - ] - ) - - def get_memory(self) -> list[MemoryItem]: - return self.memory - - def load_memory_from_file(self): - try: - with open(self.file_name) as file: - self.memory = [MemoryItem.from_dict(item) for item in json.load(file)] - logging.info(f"Memory loaded from {self.file_name} successfully.") - except FileNotFoundError: - logging.info("File not found. Starting with an empty memory.") diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py deleted file mode 100644 index a02b97dc93719..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from llama_index.packs.memary.synonym_expand.synonym import custom_synonym_expand_fn -from llama_index.packs.memary.synonym_expand.output import SynonymOutput - -__all__ = ["custom_synonym_expand_fn", "SynonymOutput"] diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py deleted file mode 100644 index 2b787eb24820e..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/output.py +++ /dev/null @@ -1,9 +0,0 @@ -from typing import List - -from langchain_core.pydantic_v1 import BaseModel, Field - - -class SynonymOutput(BaseModel): - synonyms: List[str] = Field( - description="Synonyms of keywords provided, make every letter lowercase except for the first letter" - ) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py deleted file mode 100644 index e9e46c289d7c5..0000000000000 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/synonym_expand/synonym.py +++ /dev/null @@ -1,47 +0,0 @@ -from langchain_openai import OpenAI -from langchain.prompts import PromptTemplate -from langchain_core.output_parsers import JsonOutputParser -from typing import List -import os -from llama_index.packs.memary.synonym_expand.output import SynonymOutput -from dotenv import load_dotenv - - -def custom_synonym_expand_fn(keywords: str) -> List[str]: - load_dotenv() - llm = OpenAI(openai_api_key=os.getenv("OPENAI_KEY"), temperature=0) - parser = JsonOutputParser(pydantic_object=SynonymOutput) - - template = """ - You are an expert synonym expanding system. Find synonyms or words commonly used in place to reference the same word for every word in the list: - - Some examples are: - - a synonym for Palantir may be Palantir technologies or Palantir technologies inc. - - a synonym for Austin may be Austin texas - - a synonym for Taylor swift may be Taylor - - a synonym for Winter park may be Winter park resort - - Format: {format_instructions} - - Text: {keywords} - """ - - prompt = PromptTemplate( - template=template, - input_variables=["keywords"], - partial_variables={"format_instructions": parser.get_format_instructions()}, - ) - - chain = prompt | llm | parser - result = chain.invoke({"keywords": keywords}) - - synonyms_list = [] - for category in result: - for synonym in result[category]: - synonyms_list.append(synonym.capitalize()) - - return synonyms_list - - -# testing -# print(custom_synonym_expand_fn("[Nvidia]")) From 83180478a42d939c15a7799f695496e0fcaed854 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Sun, 16 Jun 2024 23:02:07 -0500 Subject: [PATCH 19/24] refactor: follow dependencies with memary library --- .../llama-index-packs-memary/pyproject.toml | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/llama-index-packs/llama-index-packs-memary/pyproject.toml b/llama-index-packs/llama-index-packs-memary/pyproject.toml index 0632b36010f3c..ddc649f4d4527 100644 --- a/llama-index-packs/llama-index-packs-memary/pyproject.toml +++ b/llama-index-packs/llama-index-packs-memary/pyproject.toml @@ -33,33 +33,8 @@ readme = "README.md" version = "0.1.3" [tool.poetry.dependencies] -python = ">3.9.7,<3.11.10" -neo4j = "5.17.0" -python-dotenv = "1.0.1" -pyvis = "0.3.2" -streamlit = "1.31.1" -llama-index = "0.10.11" -llama-index-agent-openai = "0.1.5" -llama-index-core = "0.10.12" -llama-index-embeddings-openai = "0.1.5" -llama-index-graph-stores-nebula = "0.1.2" -llama-index-graph-stores-neo4j = "0.1.1" -llama-index-legacy = "0.9.48" -llama-index-llms-openai = "0.1.5" -llama-index-multi-modal-llms-openai = "0.1.3" -llama-index-program-openai = "0.1.3" -llama-index-question-gen-openai = "0.1.2" -llama-index-readers-file = "0.1.4" -langchain = "0.1.12" -langchain-openai = "0.0.8" -llama-index-llms-perplexity = "0.1.3" -llama-index-multi-modal-llms-ollama = "0.1.3" -llama-index-llms-ollama = "0.1.2" -pandas = "2.2.0" -geocoder = "1.38.1" -googlemaps = "4.10.0" -ansistrip = "0.1" -numpy = "1.26.4" +python = ">3.9.7,<=3.11.9" +memary = "0.1.3" ollama = "0.1.9" [tool.poetry.group.dev.dependencies] From 5057b8f89baf91b1649bd440108ac2a6c4f10849 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Sun, 16 Jun 2024 23:08:14 -0500 Subject: [PATCH 20/24] init: add MemaryChatAgentPack as base --- .../llama_index/packs/memary/base.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/base.py diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/base.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/base.py new file mode 100644 index 0000000000000..4b78d083b1e91 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/base.py @@ -0,0 +1,80 @@ +import logging +from typing import Optional, List + +from memary.agent.base_agent import Agent + + +class MemaryChatAgentPack(Agent): + """ChatAgent currently able to support Llama3 running on Ollama (default) and gpt-3.5-turbo for llm models, + and LLaVA running on Ollama (default) and gpt-4-vision-preview for the vision tool. + """ + + def __init__( + self, + name, + memory_stream_json, + entity_knowledge_store_json, + system_persona_txt, + user_persona_txt, + past_chat_json, + llm_model_name="llama3", + vision_model_name="llava", + include_from_defaults=["search", "locate", "vision", "stocks"], + ) -> None: + super().__init__( + name, + memory_stream_json, + entity_knowledge_store_json, + system_persona_txt, + user_persona_txt, + past_chat_json, + llm_model_name, + vision_model_name, + include_from_defaults, + ) + + def add_chat(self, role: str, content: str, entities: Optional[List[str]] = None): + """Add a chat to the agent's memory. + + Args: + role (str): 'system' or 'user' + content (str): content of the chat + entities (Optional[List[str]], optional): entities from Memory systems. Defaults to None. + """ + # Add a chat to the agent's memory. + self._add_contexts_to_llm_message(role, content) + + if entities: + self.memory_stream.add_memory(entities) + self.memory_stream.save_memory() + self.entity_knowledge_store.add_memory(self.memory_stream.get_memory()) + self.entity_knowledge_store.save_memory() + + self._replace_memory_from_llm_message() + self._replace_eks_to_from_message() + + def get_chat(self): + return self.contexts + + def clearMemory(self): + """Clears Neo4j database and memory stream/entity knowledge store.""" + logging.info("Deleting memory stream and entity knowledge store...") + self.memory_stream.clear_memory() + self.entity_knowledge_store.clear_memory() + + logging.info("Deleting nodes from Neo4j...") + try: + self.graph_store.query("MATCH (n) DETACH DELETE n") + except Exception as e: + logging.error(f"Error deleting nodes: {e}") + logging.info("Nodes deleted from Neo4j.") + + def _replace_memory_from_llm_message(self): + """Replace the memory_stream from the llm_message.""" + self.message.llm_message["memory_stream"] = self.memory_stream.get_memory() + + def _replace_eks_to_from_message(self): + """Replace the entity knowledge store from the llm_message. eks means entity knowledge store.""" + self.message.llm_message[ + "knowledge_entity_store" + ] = self.entity_knowledge_store.get_memory() From 2e4cfbace37bb7849c3fbc61773e8af4d9afa8da Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Sun, 16 Jun 2024 23:08:49 -0500 Subject: [PATCH 21/24] refactor: import MemaryChatAgentPack only --- .../llama_index/packs/memary/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py index af479e188e26c..a908d9cd296ce 100644 --- a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py +++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py @@ -1,5 +1,3 @@ -from llama_index.packs.memary.agent.chat_agent import ChatAgent -from llama_index.packs.memary.memory.memory_stream import MemoryStream -from llama_index.packs.memary.memory.entity_knowledge_store import EntityKnowledgeStore +from llama_index.packs.memary.base import MemaryChatAgentPack -__all__ = ["ChatAgent", "MemoryStream", "EntityKnowledgeStore"] +__all__ = ["MemaryChatAgentPack"] From fa57662d15b59b873c71ca0ed804d5aafaa345d8 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Sun, 16 Jun 2024 23:09:37 -0500 Subject: [PATCH 22/24] refactor: add test_class --- .../tests/test_packs_memary.py | 69 ++----------------- 1 file changed, 4 insertions(+), 65 deletions(-) diff --git a/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py b/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py index 15f0366fb6d96..2d89137a4c188 100644 --- a/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py +++ b/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py @@ -1,67 +1,6 @@ -import os -import unittest -from datetime import datetime -from llama_index.packs.memary import EntityKnowledgeStore, MemoryStream -from llama_index.packs.memary.memory.data_types import KnowledgeMemoryItem, MemoryItem +from llama_index.packs.memary import MemaryChatAgentPack -class TestEntityKnowledgeStore(unittest.TestCase): - def setUp(self): - self.file_name = "tests/memory/test_knowledge_memory.json" - self.entity_knowledge_store = EntityKnowledgeStore(file_name=self.file_name) - - def tearDown(self): - # Clean up test file after each test - try: - os.remove(self.file_name) - except FileNotFoundError: - pass - - def test_add_memory(self): - data = [MemoryItem("test_entity", datetime.now().replace(microsecond=0))] - self.entity_knowledge_store.add_memory(data) - assert len(self.entity_knowledge_store.knowledge_memory) == 1 - assert isinstance( - self.entity_knowledge_store.knowledge_memory[0], KnowledgeMemoryItem - ) - - def test_convert_memory_to_knowledge_memory(self): - data = [MemoryItem("test_entity", datetime.now().replace(microsecond=0))] - converted_data = ( - self.entity_knowledge_store._convert_memory_to_knowledge_memory(data) - ) - assert len(converted_data) == 1 - assert isinstance(converted_data[0], KnowledgeMemoryItem) - - def test_update_knowledge_memory(self): - data = [ - KnowledgeMemoryItem( - "knowledge_entity", 1, datetime.now().replace(microsecond=0) - ) - ] - self.entity_knowledge_store._update_knowledge_memory(data) - assert len(self.entity_knowledge_store.knowledge_memory) == 1 - assert self.entity_knowledge_store.knowledge_memory[0] == data[0] - - -class TestMemoryStream(unittest.TestCase): - def setUp(self): - self.file_name = "tests/test_memory.json" - self.memory_stream = MemoryStream(file_name=self.file_name) - - def tearDown(self): - # Clean up test file after each test - try: - os.remove(self.file_name) - except FileNotFoundError: - pass - - def test_save_and_load_memory(self): - data = [MemoryItem("test_entity", datetime.now().replace(microsecond=0))] - self.memory_stream.add_memory(data) - self.memory_stream.save_memory() - new_memory_stream = MemoryStream(file_name=self.file_name) - self.assertEqual(len(new_memory_stream), len(self.memory_stream)) - self.assertEqual( - new_memory_stream.get_memory(), self.memory_stream.get_memory() - ) +def test_class(): + names_of_base_classes = [b.__name__ for b in MemaryChatAgentPack.__mro__] + assert MemaryChatAgentPack.__name__ in names_of_base_classes From ac5e65ee790fef72e18200fe4a56c2bc6b009d74 Mon Sep 17 00:00:00 2001 From: seyeong-han Date: Sun, 16 Jun 2024 23:10:18 -0500 Subject: [PATCH 23/24] refactor: rename ChatAgent to MemaryChatAgentPack --- llama-index-packs/llama-index-packs-memary/examples/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llama-index-packs/llama-index-packs-memary/examples/app.py b/llama-index-packs/llama-index-packs-memary/examples/app.py index a060a10a3766e..bda4a1d9b9fa8 100644 --- a/llama-index-packs/llama-index-packs-memary/examples/app.py +++ b/llama-index-packs/llama-index-packs-memary/examples/app.py @@ -19,7 +19,7 @@ sys.path.append(parent_dir + "/src") -from llama_index.packs.memary.agent.chat_agent import ChatAgent +from llama_index.packs.memary.base import MemaryChatAgentPack load_dotenv() @@ -28,7 +28,7 @@ past_chat_json = "data/past_chat.json" memory_stream_json = "data/memory_stream.json" entity_knowledge_store_json = "data/entity_knowledge_store.json" -chat_agent = ChatAgent( +chat_agent = MemaryChatAgentPack( "Personal Agent", memory_stream_json, entity_knowledge_store_json, @@ -120,7 +120,7 @@ def get_models(llm_models, vision_models): ) if selected_llm_model and selected_vision_model: - chat_agent = ChatAgent( + chat_agent = MemaryChatAgentPack( "Personal Agent", memory_stream_json, entity_knowledge_store_json, From bdd97bcea04ac02ee94db01abee4c8a2624cfb00 Mon Sep 17 00:00:00 2001 From: Logan Markewich Date: Mon, 17 Jun 2024 21:40:22 -0600 Subject: [PATCH 24/24] build files --- llama-index-packs/llama-index-packs-memary/examples/BUILD | 1 + llama-index-packs/llama-index-packs-memary/tests/BUILD | 1 + 2 files changed, 2 insertions(+) create mode 100644 llama-index-packs/llama-index-packs-memary/examples/BUILD create mode 100644 llama-index-packs/llama-index-packs-memary/tests/BUILD diff --git a/llama-index-packs/llama-index-packs-memary/examples/BUILD b/llama-index-packs/llama-index-packs-memary/examples/BUILD new file mode 100644 index 0000000000000..db46e8d6c978c --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/examples/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/llama-index-packs/llama-index-packs-memary/tests/BUILD b/llama-index-packs/llama-index-packs-memary/tests/BUILD new file mode 100644 index 0000000000000..dabf212d7e716 --- /dev/null +++ b/llama-index-packs/llama-index-packs-memary/tests/BUILD @@ -0,0 +1 @@ +python_tests()