feat: added image file upload

app/api/upload/route.ts

@@ -0,0 +1,39 @@
+import { put } from "@vercel/blob";
+import { NextResponse } from "next/server";
+import { URLDetail } from "../../client/fetch/url";
+
+export async function POST(request: Request): Promise<NextResponse> {
+ const { searchParams } = new URL(request.url);
+ const filename = searchParams.get("filename");
+ if (!filename || !request.body) {
+ return NextResponse.json(
+ { error: "Missing filename URL parameter or request body" },
+ { status: 400 },
+ );
+ }
+
+ try {
+ const blob = await put(filename, request.body, {
+ access: "public",
+ });
+
+ const json = {
+ type: blob.contentType as URLDetail["type"],
+ url: blob.url,
+ // TODO: needs to return the size of the uploaded file
+ size: NaN,
+ };
+
+ return NextResponse.json<URLDetail>(json);
+ } catch (error) {
+ console.error("[Upload]", error);
+ return NextResponse.json(
+ {
+ error: (error as Error).message,
+ },
+ {
+ status: 500,
+ },
+ );
+ }
+}

app/client/fetch/file.ts

@@ -6,81 +6,67 @@ export async function getDetailContentFromFile(
 ): Promise<URLDetailContent> {
  switch (file.extension) {
  case "pdf": {
- const pdfFile = new PDFFile(file);
- return await pdfFile.getFileDetail();
+ return await getPDFFileDetail(file);
  }
  case "txt": {
- const plainTextFile = new PlainTextFile(file);
- return await plainTextFile.getFileDetail();
+ return await getTextFileDetail(file);
+ }
+ case "jpg":
+ case "jpeg": {
+ return await getImageFileDetail(file);
  }
  default: {
  throw new Error("Not supported file type");
  }
  }
 }
 
-abstract class TextFile {
- protected file: FileWrap;
- abstract getFileDetail(): Promise<URLDetailContent>;
- constructor(file: FileWrap) {
- this.file = file;
- }
-}
-
-class PDFFile extends TextFile {
- async getFileDetail() {
- const fileDataUrl = await this.file.dataURL;
- const pdfBase64 = fileDataUrl.split(",")[1];
+async function getPDFFileDetail(file: FileWrap): Promise<URLDetailContent> {
+ const fileDataUrl = await file.readData({ asURL: true });
+ const pdfBase64 = fileDataUrl.split(",")[1];
 
- const response = await fetch("/api/fetch", {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- },
- body: JSON.stringify({
- pdf: pdfBase64,
- fileName: this.file.name,
- }),
- });
- const data = await response.json();
- if (!response.ok) throw new Error(data.error);
- return data as URLDetailContent;
- }
+ const response = await fetch("/api/fetch", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ pdf: pdfBase64,
+ fileName: file.name,
+ }),
+ });
+ const data = await response.json();
+ if (!response.ok) throw new Error(data.error);
+ return data as URLDetailContent;
 }
 
-class PlainTextFile extends TextFile {
- readFileAsText(): Promise<string> {
- return new Promise((resolve, reject) => {
- const reader = new FileReader();
-
- reader.onload = (event) => {
- if (event.target) {
- resolve(event.target.result as string);
- }
- };
-
- reader.onerror = (error) => {
- reject(error);
- };
-
- reader.readAsText(this.file.file);
- });
- }
+async function getTextFileDetail(file: FileWrap): Promise<URLDetailContent> {
+ const textContent = await file.readData();
+ const response = await fetch("/api/fetch", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ text: textContent,
+ fileName: file.name,
+ }),
+ });
+ const data = await response.json();
+ if (!response.ok) throw new Error(data.error);
+ return data as URLDetailContent;
+}
 
- async getFileDetail(): Promise<URLDetailContent> {
- const textContent = await this.readFileAsText();
- const response = await fetch("/api/fetch", {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- },
- body: JSON.stringify({
- text: textContent,
- fileName: this.file.name,
- }),
- });
- const data = await response.json();
- if (!response.ok) throw new Error(data.error);
- return data as URLDetailContent;
- }
+async function getImageFileDetail(file: FileWrap) {
+ const response = await fetch(`/api/upload?filename=${file.name}`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: file.file,
+ });
+ const data = await response.json();
+ if (!response.ok) throw new Error(data.error);
+ console.log(data);
+ return data as URLDetailContent;
 }

app/client/fetch/url.ts

@@ -6,7 +6,7 @@ export type Embedding = {
 export type URLDetail = {
  url: string;
  size: number;
- type: "text/html" | "application/pdf" | "text/plain";
+ type: "text/html" | "application/pdf" | "text/plain" | "image/jpeg";
  embeddings?: Embedding[];
 };
 

app/client/platforms/llm.ts

@@ -12,9 +12,15 @@ export const MESSAGE_ROLES = [
 ] as const;
 export type MessageRole = (typeof MESSAGE_ROLES)[number];
 
+export interface Content {
+ type: "text" | "image_url";
+ text: string;
+ image_url: { url: string };
+}
+
 export interface RequestMessage {
  role: MessageRole;
- content: string;
+ content: string | Content[];
 }
 
 export const ALL_MODELS = [
@@ -36,7 +42,7 @@ export interface LLMConfig {
 }
 
 export interface ChatOptions {
- message: string;
+ message: string | Content[];
  chatHistory: RequestMessage[];
  config: LLMConfig;
  datasource?: string;

app/components/chat/chat.tsx

@@ -12,7 +12,7 @@ import { useScrollToBottom } from "@/app/hooks/useScroll";
 import { useSubmitHandler } from "@/app/hooks/useSubmit";
 import { cn } from "@/app/lib/utils";
 import { FileWrap } from "@/app/utils/file";
-import { isURL } from "@/app/client/fetch/url";
+import { URLDetail, URLDetailContent, isURL } from "@/app/client/fetch/url";
 import {
  Clipboard,
  Eraser,
@@ -44,6 +44,8 @@ import Typography from "../ui/typography";
 import { ChatAction } from "./chat-action";
 import { ClearContextDivider } from "./clear-context-divider";
 import { useBotStore } from "@/app/store/bot";
+import { getDetailContentFromFile } from "@/app/client/fetch/file";
+import Image from "next/image";
 
 const Markdown = dynamic(
  async () => (await import("../ui/markdown")).Markdown,
@@ -97,6 +99,7 @@ export function Chat() {
  const { shouldSubmit } = useSubmitHandler();
  const { scrollRef, setAutoScroll, scrollDomToBottom } = useScrollToBottom();
  const isMobileScreen = useMobileScreen();
+ const [imageFile, setImageFile] = useState<URLDetail>();
 
  // auto grow input
  const [inputRows, setInputRows] = useState(2);
@@ -130,12 +133,16 @@ export function Chat() {
  });
  };
 
- const onUserInput = async (input: string | FileWrap) => {
- const inputContent = input instanceof FileWrap ? input.name : input;
+ const call = async ({
+ input,
+ fileDetail,
+ }: {
+ input?: string;
+ fileDetail?: URLDetailContent;
+ }) => {
  await callSession(
  bot,
  session,
- inputContent,
  {
  onUpdateMessages: (messages) => {
  botStore.updateBotSession((session) => {
@@ -144,23 +151,29 @@ export function Chat() {
  }, bot.id);
  },
  },
- input instanceof FileWrap ? input : undefined,
+ input,
+ fileDetail,
  );
+ setImageFile(undefined);
+ setTemporaryURLInput("");
+ setUserInput("");
  };
 
  const doSubmitFile = async (fileInput: FileWrap) => {
- await onUserInput(fileInput);
+ const fileDetail = await getDetailContentFromFile(fileInput);
+ if (fileDetail.type === "image/jpeg") {
+ setImageFile(fileDetail);
+ } else {
+ await call({ fileDetail });
+ }
  };
 
- const doSubmit = (userInput: string) => {
- if (userInput.trim() === "") return;
- if (isURL(userInput)) {
- setTemporaryURLInput(userInput);
+ const doSubmit = async (input: string) => {
+ if (input.trim() === "") return;
+ if (isURL(input)) {
+ setTemporaryURLInput(input);
  }
- onUserInput(userInput).then(() => {
- setTemporaryURLInput("");
- });
- setUserInput("");
+ await call({ input, fileDetail: imageFile });
  if (!isMobileScreen) inputRef.current?.focus();
  setAutoScroll(true);
  };
@@ -470,6 +483,14 @@ export function Chat() {
  )}
  </div>
  <div className="flex flex-1 items-end">
+ {imageFile?.url && (
+ <Image
+ src={imageFile?.url}
+ alt="Uploaded image"
+ width={50}
+ height={50}
+ />
+ )}
  <Textarea
  className="ring-inset focus-visible:ring-offset-0 pr-28 md:pr-40 min-h-[56px]"
  ref={inputRef}

app/constant.ts

@@ -16,5 +16,5 @@ export const REQUEST_TIMEOUT_MS = 60000;
 export const CHAT_PAGE_SIZE = 15;
 export const MAX_RENDER_MSG_COUNT = 45;
 
-export const ALLOWED_DOCUMENT_EXTENSIONS = ["pdf", "txt"];
+export const ALLOWED_DOCUMENT_EXTENSIONS = ["pdf", "txt", "jpeg", "jpg"];
 export const DOCUMENT_FILE_SIZE_LIMIT = 1024 * 1024 * 10; // 10 MB