fix: cleaned up root and regenerated data sources with overlap

.env.template

@@ -1,22 +1,6 @@
-
 # Your openai api key. (required)
 OPENAI_API_KEY=sk-xxxx
 
-# Override openai api request base url. (optional)
-# Default: https://api.openai.com
-# Examples: http:https://your-openai-proxy.com
-BASE_URL=
-
-# Specify OpenAI organization ID.(optional)
-# Default: Empty
-# If you do not want users to input their own API key, set this value to 1.
-OPENAI_ORG_ID=
-
-# (optional)
-# Default: Empty
-# If you do not want users to use GPT-4, set this value to 1.
-DISABLE_GPT4=
-
 # see https://next-auth.js.org/providers/linkedin
 LINKEDIN_CLIENT_ID=
 LINKEDIN_CLIENT_SECRET=

README.md

@@ -29,17 +29,6 @@ Unc is enterprise-ready, featuring:
 - Supports [Azure OpenAI from Microsoft](https://azure.microsoft.com/en-us/products/ai-services/openai-service), eliminating traffic to OpenAI.
 - Privacy first; personal data is stored locally in the browser.
 
-## 💼 Enterprise
-
-A typical enterprise installation requires:
-
-- Custom user authentication (e.g. Active Directory).
-- Installation in a corporate cloud (AWS or Azure).
-- Integration with data from third-party systems (e.g., Salesforce, ServiceNow, SAP, Oracle).
-
-For enterprise support and installations, reach out to me on
-[LinkedIn](https://linkedin.com/marcusschiesser).
-
 ## ⚡️ Quick start
 
 ### Use Gitpod
@@ -60,12 +49,15 @@ cd unc
 - Run the dev server
 
 ```bash
-yarn install
-yarn dev
+pnpm install
+pnpm dev
 ```
 
-## ❤️ Community
+### Recreate Storage
 
-Join the Unc community on [GitHub Discussions](https://github.com/marcusschiesser/unc/discussions). You can ask questions, share ideas, and showcase your projects.
-
-Feedback is paramount. Please do share any suggestions, ideas, or bugs you encounter via [GitHub issues](https://github.com/marcusschiesser/unc/issues). Your insight is invaluable for enhancing future releases.
+The app is using a [`ChatEngine`](https://ts.llamaindex.ai/modules/high_level/chat_engine) for each bot with a different [`VectorStoreIndex`](https://ts.llamaindex.ai/modules/high_level/data_index) attached.
+The `cache` folder in the file system is used as [Storage](The https://ts.llamaindex.ai/modules/low_level/storage) for the `VectorStoreIndex`. To re-create the storage 
+Vector Indexes 
+```bash
+pnpm run generate
+```

app/api/llm/route.ts

@@ -14,7 +14,10 @@ import {
 import { NextRequest, NextResponse } from "next/server";
 import { LLMConfig } from "../../client/platforms/llm";
 import { getDataSource } from "./datasource";
-import { DATASOURCES_CHUNK_SIZE } from "@/scripts/constants.mjs";
+import {
+ DATASOURCES_CHUNK_OVERLAP,
+ DATASOURCES_CHUNK_SIZE,
+} from "@/scripts/constants.mjs";
 import { Embedding } from "@/app/client/fetch/url";
 
 async function createChatEngine(
@@ -105,6 +108,7 @@ export async function POST(request: NextRequest) {
  const serviceContext = serviceContextFromDefaults({
  llm,
  chunkSize: DATASOURCES_CHUNK_SIZE,
+ chunkOverlap: DATASOURCES_CHUNK_OVERLAP,
  });
 
  const chatEngine = await createChatEngine(

package.json

@@ -4,7 +4,7 @@
  "license": "AGPL-3.0-or-later",
  "scripts": {
  "dev": "next dev",
- "build": "cross-env next build",
+ "build": "next build",
  "start": "next start",
  "lint": "next lint",
  "format:check": "prettier --check --ignore-path .gitignore app",
@@ -79,7 +79,6 @@
  "@types/react": "^18.2.14",
  "@types/react-dom": "^18.2.7",
  "@types/react-katex": "^3.0.0",
- "cross-env": "^7.0.3",
  "eslint": "^8.44.0",
  "eslint-config-next": "13.4.13",
  "eslint-config-prettier": "^9.0.0",

scripts/generate.mjs

@@ -10,6 +10,7 @@ import {
  DATASOURCES_CACHE_DIR,
  DATASOURCES_DIR,
  DATASOURCES_CHUNK_SIZE,
+ DATASOURCES_CHUNK_OVERLAP,
 } from "./constants.mjs";
 
 async function getRuntime(func) {
@@ -44,6 +45,7 @@ async function generateDatasource(serviceContext, datasource) {
 (async () => {
  const serviceContext = serviceContextFromDefaults({
  chunkSize: DATASOURCES_CHUNK_SIZE,
+ chunkOverlap: DATASOURCES_CHUNK_OVERLAP,
  });
 
  for (const datasource of DATASOURCES) {