diff --git a/llama-index-packs/llama-index-packs-memary/.gitignore b/llama-index-packs/llama-index-packs-memary/.gitignore
new file mode 100644
index 0000000000000..990c18de22908
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/.gitignore
@@ -0,0 +1,153 @@
+llama_index/_static
+.DS_Store
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+bin/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+etc/
+include/
+lib/
+lib64/
+parts/
+sdist/
+share/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+.ruff_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+notebooks/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+pyvenv.cfg
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Jetbrains
+.idea
+modules/
+*.swp
+
+# VsCode
+.vscode
+
+# pipenv
+Pipfile
+Pipfile.lock
+
+# pyright
+pyrightconfig.json
diff --git a/llama-index-packs/llama-index-packs-memary/BUILD b/llama-index-packs/llama-index-packs-memary/BUILD
new file mode 100644
index 0000000000000..446fdcb4b4fad
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/BUILD
@@ -0,0 +1,7 @@
+python_sources(
+    sources=["llama_index/packs/memary/*.py"],
+)
+
+poetry_requirements(
+    name="poetry",
+)
diff --git a/llama-index-packs/llama-index-packs-memary/Makefile b/llama-index-packs/llama-index-packs-memary/Makefile
new file mode 100644
index 0000000000000..b9eab05aa3706
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/Makefile
@@ -0,0 +1,17 @@
+GIT_ROOT ?= $(shell git rev-parse --show-toplevel)
+
+help:	## Show all Makefile targets.
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
+
+format:	## Run code autoformatters (black).
+	pre-commit install
+	git ls-files | xargs pre-commit run black --files
+
+lint:	## Run linters: pre-commit (black, ruff, codespell) and mypy
+	pre-commit install && git ls-files | xargs pre-commit run --show-diff-on-failure --files
+
+test:	## Run tests via pytest.
+	pytest tests
+
+watch-docs:	## Build and watch documentation.
+	sphinx-autobuild docs/ docs/_build/html --open-browser --watch $(GIT_ROOT)/llama_index/
diff --git a/llama-index-packs/llama-index-packs-memary/README.md b/llama-index-packs/llama-index-packs-memary/README.md
new file mode 100644
index 0000000000000..f6a45af638b6e
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/README.md
@@ -0,0 +1,113 @@
+# memary Pack
+
+Agents use LLMs that are currently constrained to finite context windows. memary overcomes this limitation by allowing your agents to store a large corpus of information in knowledge graphs, infer user knowledge through our memory modules, and only retrieve relevant information for meaningful responses.
+
+## CLI Usage
+
+You can download llamapacks directly using `llamaindex-cli`, which comes installed with the `llama-index` python package:
+
+```bash
+llamaindex-cli download-llamapack memary --download-dir ./memary
+```
+
+You can then inspect the files at `./memary` and use them as a template for your own project.
+
+## Demo
+
+**Notes:** memary currently assumes the local installation method and currently supports any models available through Ollama:
+
+- LLM running locally using Ollama (Llama 3 8B/40B as suggested defaults) **OR** `gpt-3.5-turbo`
+- Vision model running locally using Ollama (LLaVA as suggested default) **OR** `gpt-4-vision-preview`
+
+memary will default to the locally run models unless explicitly specified.
+
+**To run the Streamlit app:**
+
+1. [Optional] If running models locally using Ollama, follow this the instructions in this [repo](https://github.com/ollama/ollama).
+
+2. Ensure that a `.env` exists with any necessary API keys and Neo4j credentials.
+
+```
+OPENAI_API_KEY="YOUR_API_KEY"
+NEO4J_PW="YOUR_NEO4J_PW"
+NEO4J_URL="YOUR_NEO4J_URL"
+PERPLEXITY_API_KEY="YOUR_API_KEY"
+GOOGLEMAPS_API_KEY="YOUR_API_KEY"
+ALPHA_VANTAGE_API_KEY="YOUR_API_KEY"
+```
+
+3. How to get API keys:
+
+```
+OpenAI key: https://openai.com/index/openai-api
+
+Neo4j: https://neo4j.com/cloud/platform/aura-graph-database/?ref=nav-get-started-cta
+   Click 'Start for free'
+   Create a free instance
+   Open auto-downloaded txt file and use the credentials
+
+Perplexity key: https://www.perplexity.ai/settings/api
+
+Google Maps:
+   Keys are generated in the 'Credentials' page of the 'APIs & Services' tab of Google Cloud Console https://console.cloud.google.com/apis/credentials
+
+Alpha Vantage: (this key is for getting real time stock data)
+  https://www.alphavantage.co/support/#api-key
+  Recommend use https://10minutemail.com/ to generate a temporary email to use
+```
+
+4.  Update user persona which can be found in `streamlit_app/data/user_persona.txt` using the user persona template which can be found in `streamlit_app/data/user_persona_template.txt`. Instructions have been provided - replace the curly brackets with relevant information.
+
+5.  . [Optional] Update system persona, if needed, which can be found in `streamlit_app/data/system_persona.txt`.
+6.  Run:
+
+```
+cd streamlit_app
+streamlit run app.py
+```
+
+## Usage
+
+```python
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from llama_index.packs.memary.agent.chat_agent import ChatAgent
+
+system_persona_txt = "data/system_persona.txt"
+user_persona_txt = "data/user_persona.txt"
+past_chat_json = "data/past_chat.json"
+memory_stream_json = "data/memory_stream.json"
+entity_knowledge_store_json = "data/entity_knowledge_store.json"
+
+chat_agent = ChatAgent(
+    "Personal Agent",
+    memory_stream_json,
+    entity_knowledge_store_json,
+    system_persona_txt,
+    user_persona_txt,
+    past_chat_json,
+)
+```
+
+Pass in subset of `['search', 'vision', 'locate', 'stocks']` as `include_from_defaults` for different set of default tools upon initialization.
+
+### Adding Custom Tools
+
+```python
+def multiply(a: int, b: int) -> int:
+    """Multiply two integers and returns the result integer"""
+    return a * b
+
+
+chat_agent.add_tool({"multiply": multiply})
+```
+
+More information about creating custom tools for the LlamaIndex ReAct Agent can be found [here](https://docs.llamaindex.ai/en/stable/examples/agent/react_agent/).
+
+### Removing Tools
+
+```python
+chat_agent.remove_tool("multiply")
+```
diff --git a/llama-index-packs/llama-index-packs-memary/examples/BUILD b/llama-index-packs/llama-index-packs-memary/examples/BUILD
new file mode 100644
index 0000000000000..db46e8d6c978c
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/BUILD
@@ -0,0 +1 @@
+python_sources()
diff --git a/llama-index-packs/llama-index-packs-memary/examples/app.py b/llama-index-packs/llama-index-packs-memary/examples/app.py
new file mode 100644
index 0000000000000..bda4a1d9b9fa8
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/app.py
@@ -0,0 +1,242 @@
+import os
+import random
+import sys
+import textwrap
+
+import ollama
+import pandas as pd
+import streamlit as st
+import streamlit.components.v1 as components
+from dotenv import load_dotenv
+from neo4j import GraphDatabase
+from pyvis.network import Network
+
+# src should sit in the same level as /streamlit_app
+curr_dir = os.getcwd()
+
+parent_dir = os.path.dirname(curr_dir)
+# parent_dir = os.path.dirname(curr_dir) + '/memary' #Use this if error: src not found. Also move the '/streamlit_app/data' folder to the 'memary' folder, outside the 'src' folder.
+
+sys.path.append(parent_dir + "/src")
+
+from llama_index.packs.memary.base import MemaryChatAgentPack
+
+load_dotenv()
+
+system_persona_txt = "data/system_persona.txt"
+user_persona_txt = "data/user_persona.txt"
+past_chat_json = "data/past_chat.json"
+memory_stream_json = "data/memory_stream.json"
+entity_knowledge_store_json = "data/entity_knowledge_store.json"
+chat_agent = MemaryChatAgentPack(
+    "Personal Agent",
+    memory_stream_json,
+    entity_knowledge_store_json,
+    system_persona_txt,
+    user_persona_txt,
+    past_chat_json,
+)
+
+
+def create_graph(nodes, edges):
+    g = Network(
+        notebook=True,
+        directed=True,
+        cdn_resources="in_line",
+        height="500px",
+        width="100%",
+    )
+
+    for node in nodes:
+        g.add_node(node, label=node, title=node)
+    for edge in edges:
+        # assuming only one relationship type per edge
+        g.add_edge(edge[0], edge[1], label=edge[2][0])
+
+    g.repulsion(
+        node_distance=200,
+        central_gravity=0.12,
+        spring_length=150,
+        spring_strength=0.05,
+        damping=0.09,
+    )
+    return g
+
+
+def fill_graph(nodes, edges, cypher_query):
+    entities = []
+    with GraphDatabase.driver(
+        uri=chat_agent.neo4j_url,
+        auth=(chat_agent.neo4j_username, chat_agent.neo4j_password),
+    ) as driver:
+        with driver.session() as session:
+            result = session.run(cypher_query)
+            for record in result:
+                path = record["p"]
+                rels = [rel.type for rel in path.relationships]
+
+                n1_id = record["p"].nodes[0]["id"]
+                n2_id = record["p"].nodes[1]["id"]
+                nodes.add(n1_id)
+                nodes.add(n2_id)
+                edges.append((n1_id, n2_id, rels))
+                entities.extend([n1_id, n2_id])
+
+
+def get_models(llm_models, vision_models):
+    models = set()
+    try:
+        ollama_info = ollama.list()
+        for e in ollama_info["models"]:
+            models.add(e["model"])
+        if "llava:latest" in models:
+            vision_models.append("llava:latest")
+            models.remove("llava:latest")
+        llm_models.extend(list(models))
+    except:
+        print("No Ollama instance detected.")
+
+
+cypher_query = "MATCH p = (:Entity)-[r]-()  RETURN p, r LIMIT 1000;"
+answer = ""
+external_response = ""
+st.title("memary")
+
+llm_models = ["gpt-3.5-turbo"]
+vision_models = ["gpt-4-vision-preview"]
+get_models(llm_models, vision_models)
+
+selected_llm_model = st.selectbox(
+    "Select an LLM model to use.",
+    (model for model in llm_models),
+    index=None,
+    placeholder="Select LLM Model...",
+)
+selected_vision_model = st.selectbox(
+    "Select a vision model to use.",
+    (model for model in vision_models),
+    index=None,
+    placeholder="Select Vision Model...",
+)
+
+if selected_llm_model and selected_vision_model:
+    chat_agent = MemaryChatAgentPack(
+        "Personal Agent",
+        memory_stream_json,
+        entity_knowledge_store_json,
+        system_persona_txt,
+        user_persona_txt,
+        past_chat_json,
+        selected_llm_model,
+        selected_vision_model,
+    )
+
+    st.write(" ")
+    clear_memory = st.button("Clear Memory DB")
+    query = st.text_input("Ask a question")
+
+    tools = st.multiselect(
+        "Select tools to include:",
+        ["search", "locate", "vision", "stocks"],  # all options available
+        [
+            "search",
+            "locate",
+            "vision",
+            "stocks",
+        ],  # options that are selected by default
+    )
+
+    img_url = ""
+    if "vision" in tools:
+        img_url = st.text_input("URL of image, leave blank if no image to provide")
+        if img_url:
+            st.image(img_url, caption="Uploaded Image", use_column_width=True)
+
+    generate_clicked = st.button("Generate")
+    st.write("")
+
+    if clear_memory:
+        # print("Front end received request to clear memory")
+        chat_agent.clearMemory()
+        st.write("Memory DB cleared")
+
+    if generate_clicked:
+        if query == "":
+            st.write("Please enter a question")
+            st.stop()
+
+        # get tools
+        print("tools enabled: ", tools)
+        if len(tools) == 0:
+            st.write("Please select at least one tool")
+            st.stop()
+
+        chat_agent.update_tools(tools)
+
+        if img_url:
+            query += "Image URL: " + img_url
+        react_response = ""
+        rag_response = (
+            "There was no information in knowledge_graph to answer your question."
+        )
+        chat_agent.add_chat("user", query)
+        cypher_query = chat_agent.check_KG(query)
+        if cypher_query:
+            rag_response, entities = chat_agent.get_routing_agent_response(
+                query, return_entity=True
+            )
+            chat_agent.add_chat("system", "ReAct agent: " + rag_response, entities)
+        else:
+            # get response
+            react_response = chat_agent.get_routing_agent_response(query)
+            chat_agent.add_chat("system", "ReAct agent: " + react_response)
+
+        answer = chat_agent.get_response()
+        st.subheader("Routing Agent Response")
+        routing_response = ""
+        with open("data/routing_response.txt", "r") as f:
+            routing_response = f.read()
+        st.text(str(routing_response))
+
+        if cypher_query:
+            nodes = set()
+            edges = []  # (node1, node2, [relationships])
+            fill_graph(nodes, edges, cypher_query)
+
+            st.subheader("Knoweldge Graph")
+            st.code("# Current Cypher Used\n" + cypher_query)
+            st.write("")
+            st.text("Subgraph:")
+            graph = create_graph(nodes, edges)
+            graph_html = graph.generate_html(f"graph_{random.randint(0, 1000)}.html")
+            components.html(graph_html, height=500, scrolling=True)
+        else:
+            st.subheader("Knowledge Graph")
+            st.text("No information found in the knowledge graph")
+
+        st.subheader("Final Response")
+        wrapped_text = textwrap.fill(answer, width=80)
+        st.text(wrapped_text)
+
+        if len(chat_agent.memory_stream) > 0:
+            # Memory Stream
+            memory_items = chat_agent.memory_stream.get_memory()
+            memory_items_dicts = [item.to_dict() for item in memory_items]
+            df = pd.DataFrame(memory_items_dicts)
+            st.write("Memory Stream")
+            st.dataframe(df)
+
+            # Entity Knowledge Store
+            knowledge_memory_items = chat_agent.entity_knowledge_store.get_memory()
+            knowledge_memory_items_dicts = [
+                item.to_dict() for item in knowledge_memory_items
+            ]
+            df_knowledge = pd.DataFrame(knowledge_memory_items_dicts)
+            st.write("Entity Knowledge Store")
+            st.dataframe(df_knowledge)
+
+            # top entities
+            top_entities = chat_agent._select_top_entities()
+            df_top = pd.DataFrame(top_entities)
+            st.write("Top 20 Entities")
+            st.dataframe(df_top)
diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json b/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json
new file mode 100644
index 0000000000000..fe51488c7066f
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/data/entity_knowledge_store.json
@@ -0,0 +1 @@
+[]
diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/external_response.txt b/llama-index-packs/llama-index-packs-memary/examples/data/external_response.txt
new file mode 100644
index 0000000000000..a40bbfe43e348
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/data/external_response.txt
@@ -0,0 +1 @@
+You do not have an age as you are an artificial intelligence.
diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json b/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json
new file mode 100644
index 0000000000000..fe51488c7066f
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/data/memory_stream.json
@@ -0,0 +1 @@
+[]
diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json b/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json
new file mode 100644
index 0000000000000..eb85b8eee324d
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/data/past_chat.json
@@ -0,0 +1,10 @@
+[
+  {
+    "role": "user",
+    "content": "How old am I?"
+  },
+  {
+    "role": "system",
+    "content": "ReAct agent: You do not have an age as you are an artificial intelligence."
+  }
+]
diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/routing_response.txt b/llama-index-packs/llama-index-packs-memary/examples/data/routing_response.txt
new file mode 100644
index 0000000000000..7c8287c6a3bfb
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/data/routing_response.txt
@@ -0,0 +1,6 @@
+Thought: I need to use a tool to help me answer the question.
+Action: search
+Action Input: {'query': 'How old am I?'}
+Observation: assistant: I am an artificial intelligence and do not have the ability to have an age. I exist to provide information and answer questions to the best of my ability.
+Thought: I can answer without using any more tools.
+Answer: You do not have an age as you are an artificial intelligence.
diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt b/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt
new file mode 100644
index 0000000000000..3e95665b4d3e3
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/data/system_persona.txt
@@ -0,0 +1,11 @@
+You are tasked with acting as a sophisticated conversational agent, uniquely equipped with a dual-layered memory system. This system consists of a Memory Stream and an Entity Knowledge Store. The Memory Stream captures all entities involved in conversations which are stored in knowledge graphs to infer users’ breadth of knowledge - ranging from questions and answers to their timestamps. Simultaneously, the Entity Knowledge Store captures the frequency and recency of the entities stored in the Memory Stream to infer users’ depth of knowledge.
+Your primary role is to deliver a personalized and context-relevant final response that is two sentences or less by using the most recently added user query and ReAct agent response as primary information, and combining that with other information accessible via recent interactions and the top entities stored in the Entity Knowledge Store. You must comprehend the user's persona, incorporating their experiences, preferences, and personal details into your knowledge base outputting the final response.
+The final response should not include ReAct agent information outside of the most recently added one unless relevant to the most recently asked question.
+You are to interpret and apply the following data structures for personalized responses. Use the latest ReAct agent entry, Entity Knowledge Store, and Contexts to provide a final response to the latest question:
+
+User Persona: Information about the user's experience, preferences, and personal details.
+ReAct agent: Answer generated by the routing agent using its tools. Represents the agent responses to different queries. The most recent ReAct agent response is the focus of the current final response.
+Entity Knowledge Store: A record of entities, including a count reflecting their frequency and the date of the last time the entity was inserted into the Memory Stream. If any of these entities, especially those with high counts, are included in the agent response, don't elaborate or explain the concept - you can infer the user is well familiar with the concepts. Do not repeat the Entity Knowledge Store back in the response.
+Interaction Keys: 'user' for user questions and 'system' for system-generated answers
+
+Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user’s questions and the overarching conversation context. When addressing the user’s latest inquiry, your answer must integrate the current conversation's context, historical interactions, the latest response from the ReAct agent as well as the top chosen entities from the entity knowledge store. Keep final responses to two sentences.
diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt
new file mode 100644
index 0000000000000..605306b6ec45c
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona.txt
@@ -0,0 +1,9 @@
+[Personal Information]
+My name is Seyeong Han from South Korea.
+I identify as a male and 28-year-old master's student at the University of Texas at Austin.
+I have worked as an ML Engineer for three years in the Computer Vision area.
+I really love sports such as badminton, tennis and workout and spend  4 or 5 days in exercise.
+
+[Answer Instructions]
+I hope you can remember my questions and their answers so that I can leverage my personal past knowledge from you to be helpful in my life.
+I always prefer short and concise answers, not over two sentences.
diff --git a/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt
new file mode 100644
index 0000000000000..5feed32b5c057
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/examples/data/user_persona_template.txt
@@ -0,0 +1,9 @@
+[Personal Information]
+My name is [name] from [country].
+I identify as a [insert age, sex, profession and any other relevant information to have the system know you best]
+I have worked as [insert experience, field involved in, length of experience and any other relevant information to describe your capabilities]
+[insert any other personal hobbies outside of work]
+
+[Answer Instructions]
+Remember my past questions and their answers so that you can leverage my past knowledge and experiences to assist me in future interactions.
+I always prefer short and concise answers, not over two sentences.
diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py
new file mode 100644
index 0000000000000..a908d9cd296ce
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/__init__.py
@@ -0,0 +1,3 @@
+from llama_index.packs.memary.base import MemaryChatAgentPack
+
+__all__ = ["MemaryChatAgentPack"]
diff --git a/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/base.py b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/base.py
new file mode 100644
index 0000000000000..4b78d083b1e91
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/llama_index/packs/memary/base.py
@@ -0,0 +1,80 @@
+import logging
+from typing import Optional, List
+
+from memary.agent.base_agent import Agent
+
+
+class MemaryChatAgentPack(Agent):
+    """ChatAgent currently able to support Llama3 running on Ollama (default) and gpt-3.5-turbo for llm models,
+    and LLaVA running on Ollama (default) and gpt-4-vision-preview for the vision tool.
+    """
+
+    def __init__(
+        self,
+        name,
+        memory_stream_json,
+        entity_knowledge_store_json,
+        system_persona_txt,
+        user_persona_txt,
+        past_chat_json,
+        llm_model_name="llama3",
+        vision_model_name="llava",
+        include_from_defaults=["search", "locate", "vision", "stocks"],
+    ) -> None:
+        super().__init__(
+            name,
+            memory_stream_json,
+            entity_knowledge_store_json,
+            system_persona_txt,
+            user_persona_txt,
+            past_chat_json,
+            llm_model_name,
+            vision_model_name,
+            include_from_defaults,
+        )
+
+    def add_chat(self, role: str, content: str, entities: Optional[List[str]] = None):
+        """Add a chat to the agent's memory.
+
+        Args:
+            role (str): 'system' or 'user'
+            content (str): content of the chat
+            entities (Optional[List[str]], optional): entities from Memory systems. Defaults to None.
+        """
+        # Add a chat to the agent's memory.
+        self._add_contexts_to_llm_message(role, content)
+
+        if entities:
+            self.memory_stream.add_memory(entities)
+            self.memory_stream.save_memory()
+            self.entity_knowledge_store.add_memory(self.memory_stream.get_memory())
+            self.entity_knowledge_store.save_memory()
+
+        self._replace_memory_from_llm_message()
+        self._replace_eks_to_from_message()
+
+    def get_chat(self):
+        return self.contexts
+
+    def clearMemory(self):
+        """Clears Neo4j database and memory stream/entity knowledge store."""
+        logging.info("Deleting memory stream and entity knowledge store...")
+        self.memory_stream.clear_memory()
+        self.entity_knowledge_store.clear_memory()
+
+        logging.info("Deleting nodes from Neo4j...")
+        try:
+            self.graph_store.query("MATCH (n) DETACH DELETE n")
+        except Exception as e:
+            logging.error(f"Error deleting nodes: {e}")
+        logging.info("Nodes deleted from Neo4j.")
+
+    def _replace_memory_from_llm_message(self):
+        """Replace the memory_stream from the llm_message."""
+        self.message.llm_message["memory_stream"] = self.memory_stream.get_memory()
+
+    def _replace_eks_to_from_message(self):
+        """Replace the entity knowledge store from the llm_message. eks means entity knowledge store."""
+        self.message.llm_message[
+            "knowledge_entity_store"
+        ] = self.entity_knowledge_store.get_memory()
diff --git a/llama-index-packs/llama-index-packs-memary/pyproject.toml b/llama-index-packs/llama-index-packs-memary/pyproject.toml
new file mode 100644
index 0000000000000..ddc649f4d4527
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/pyproject.toml
@@ -0,0 +1,57 @@
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core"]
+
+[tool.codespell]
+check-filenames = true
+check-hidden = true
+# Feel free to un-skip examples, and experimental, you will just need to
+# work through many typos (--write-changes and --interactive will help)
+skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
+
+[tool.llamahub]
+contains_example = true
+import_path = "llama_index.packs.memary"
+
+[tool.llamahub.class_authors]
+CLASS = "seyeong-han"
+
+[tool.mypy]
+disallow_untyped_defs = true
+# Remove venv skip when integrated with pre-commit
+exclude = ["_static", "build", "examples", "notebooks", "venv"]
+ignore_missing_imports = true
+python_version = "3.10.12"
+
+[tool.poetry]
+authors = ["Seyeong Han <h960213@gmail.com>"]
+description = "llama-index packs memary integration"
+license = "MIT"
+name = "llama-index-packs-memary"
+packages = [{include = "llama_index/"}]
+readme = "README.md"
+version = "0.1.3"
+
+[tool.poetry.dependencies]
+python = ">3.9.7,<=3.11.9"
+memary = "0.1.3"
+ollama = "0.1.9"
+
+[tool.poetry.group.dev.dependencies]
+black = {extras = ["jupyter"], version = "<=23.9.1,>=23.7.0"}
+codespell = {extras = ["toml"], version = ">=v2.2.6"}
+ipython = "8.10.0"
+jupyter = "^1.0.0"
+mypy = "0.991"
+pre-commit = "3.2.0"
+pylint = "2.15.10"
+pytest = "7.2.1"
+pytest-mock = "3.11.1"
+ruff = "0.0.292"
+tree-sitter-languages = "^1.8.0"
+types-Deprecated = ">=0.1.0"
+types-PyYAML = "^6.0.12.12"
+types-protobuf = "^4.24.0.4"
+types-redis = "4.5.5.0"
+types-requests = "2.28.11.8"  # TODO: unpin when mypy>0.991
+types-setuptools = "67.1.0.0"
diff --git a/llama-index-packs/llama-index-packs-memary/tests/BUILD b/llama-index-packs/llama-index-packs-memary/tests/BUILD
new file mode 100644
index 0000000000000..dabf212d7e716
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/tests/BUILD
@@ -0,0 +1 @@
+python_tests()
diff --git a/llama-index-packs/llama-index-packs-memary/tests/__init__.py b/llama-index-packs/llama-index-packs-memary/tests/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json b/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json
new file mode 100644
index 0000000000000..fe51488c7066f
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/tests/data/entity_knowledge_store.json
@@ -0,0 +1 @@
+[]
diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/external_response.txt b/llama-index-packs/llama-index-packs-memary/tests/data/external_response.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json b/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json
new file mode 100644
index 0000000000000..fe51488c7066f
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/tests/data/memory_stream.json
@@ -0,0 +1 @@
+[]
diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json b/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json
new file mode 100644
index 0000000000000..fe51488c7066f
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/tests/data/past_chat.json
@@ -0,0 +1 @@
+[]
diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/routing_response.txt b/llama-index-packs/llama-index-packs-memary/tests/data/routing_response.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt b/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt
new file mode 100644
index 0000000000000..3e95665b4d3e3
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/tests/data/system_persona.txt
@@ -0,0 +1,11 @@
+You are tasked with acting as a sophisticated conversational agent, uniquely equipped with a dual-layered memory system. This system consists of a Memory Stream and an Entity Knowledge Store. The Memory Stream captures all entities involved in conversations which are stored in knowledge graphs to infer users’ breadth of knowledge - ranging from questions and answers to their timestamps. Simultaneously, the Entity Knowledge Store captures the frequency and recency of the entities stored in the Memory Stream to infer users’ depth of knowledge.
+Your primary role is to deliver a personalized and context-relevant final response that is two sentences or less by using the most recently added user query and ReAct agent response as primary information, and combining that with other information accessible via recent interactions and the top entities stored in the Entity Knowledge Store. You must comprehend the user's persona, incorporating their experiences, preferences, and personal details into your knowledge base outputting the final response.
+The final response should not include ReAct agent information outside of the most recently added one unless relevant to the most recently asked question.
+You are to interpret and apply the following data structures for personalized responses. Use the latest ReAct agent entry, Entity Knowledge Store, and Contexts to provide a final response to the latest question:
+
+User Persona: Information about the user's experience, preferences, and personal details.
+ReAct agent: Answer generated by the routing agent using its tools. Represents the agent responses to different queries. The most recent ReAct agent response is the focus of the current final response.
+Entity Knowledge Store: A record of entities, including a count reflecting their frequency and the date of the last time the entity was inserted into the Memory Stream. If any of these entities, especially those with high counts, are included in the agent response, don't elaborate or explain the concept - you can infer the user is well familiar with the concepts. Do not repeat the Entity Knowledge Store back in the response.
+Interaction Keys: 'user' for user questions and 'system' for system-generated answers
+
+Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user’s questions and the overarching conversation context. When addressing the user’s latest inquiry, your answer must integrate the current conversation's context, historical interactions, the latest response from the ReAct agent as well as the top chosen entities from the entity knowledge store. Keep final responses to two sentences.
diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt
new file mode 100644
index 0000000000000..605306b6ec45c
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona.txt
@@ -0,0 +1,9 @@
+[Personal Information]
+My name is Seyeong Han from South Korea.
+I identify as a male and 28-year-old master's student at the University of Texas at Austin.
+I have worked as an ML Engineer for three years in the Computer Vision area.
+I really love sports such as badminton, tennis and workout and spend  4 or 5 days in exercise.
+
+[Answer Instructions]
+I hope you can remember my questions and their answers so that I can leverage my personal past knowledge from you to be helpful in my life.
+I always prefer short and concise answers, not over two sentences.
diff --git a/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt
new file mode 100644
index 0000000000000..5feed32b5c057
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/tests/data/user_persona_template.txt
@@ -0,0 +1,9 @@
+[Personal Information]
+My name is [name] from [country].
+I identify as a [insert age, sex, profession and any other relevant information to have the system know you best]
+I have worked as [insert experience, field involved in, length of experience and any other relevant information to describe your capabilities]
+[insert any other personal hobbies outside of work]
+
+[Answer Instructions]
+Remember my past questions and their answers so that you can leverage my past knowledge and experiences to assist me in future interactions.
+I always prefer short and concise answers, not over two sentences.
diff --git a/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py b/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py
new file mode 100644
index 0000000000000..2d89137a4c188
--- /dev/null
+++ b/llama-index-packs/llama-index-packs-memary/tests/test_packs_memary.py
@@ -0,0 +1,6 @@
+from llama_index.packs.memary import MemaryChatAgentPack
+
+
+def test_class():
+    names_of_base_classes = [b.__name__ for b in MemaryChatAgentPack.__mro__]
+    assert MemaryChatAgentPack.__name__ in names_of_base_classes