add audio

app/api/capture/route.ts

@@ -2,7 +2,8 @@ import { openai } from "@/app/openai";
 
 export async function GET() {
  // Add a cache-busting parameter to the URL
- const url = `http:https://192.168.86.48:5000/capture?_=${new Date().getTime()}`;
+ // get base url from env var $SERVER_URL
+ const url = `${process.env.NEXT_PUBLIC_SERVER_URL}/capture?_=${new Date().getTime()}`;
 
  // Read image from remote camera
  const capture = await fetch(url);

app/api/whisper/route.ts

@@ -0,0 +1,42 @@
+const FormData = require('form-data');
+import { withFileUpload } from 'next-multiparty';
+import { createReadStream } from 'fs';
+
+export const config = {
+ api: {
+ bodyParser: false,
+ },
+};
+
+export default withFileUpload(async (req, res) => {
+ const file = req.file;
+ if (!file) {
+ res.status(400).send('No file uploaded');
+ return;
+ }
+
+ // Create form data
+ const formData = new FormData();
+ formData.append('file', createReadStream(file.filepath), 'audio.wav');
+ formData.append('model', 'whisper-1');
+ const response = await fetch(
+ 'https://api.openai.com/v1/audio/transcriptions',
+ {
+ method: 'POST',
+ headers: {
+ ...formData.getHeaders(),
+ Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+ },
+ body: formData,
+ }
+ );
+
+ const { text, error } = await response.json();
+ if (response.ok) {
+ res.status(200).json({ text: text });
+ } else {
+ console.log('OPEN AI ERROR:');
+ console.log(error.message);
+ res.status(400).send(new Error());
+ }
+});

app/components/camera-feed.tsx

@@ -4,7 +4,8 @@ import React, { useEffect, useState } from 'react';
 import styles from './camera-feed.module.css';
 
 const CameraFeed = () => {
- const [imageSrc, setImageSrc] = useState('http:https://192.168.86.48:5000/video_feed');
+ console.log("server url is", process.env.NEXT_PUBLIC_SERVER_URL)
+ const [imageSrc, setImageSrc] = useState(`${process.env.NEXT_PUBLIC_SERVER_URL}/video_feed`);
 
  return (
  <div className={styles.cameraFeed}>

app/components/chat.tsx

@@ -7,6 +7,7 @@ import Markdown from "react-markdown";
 // @ts-expect-error - no types for this yet
 import { AssistantStreamEvent } from "openai/resources/beta/assistants/assistants";
 import { RequiredActionFunctionToolCall } from "openai/resources/beta/threads/runs/runs";
+import { AudioRecorder } from 'react-audio-voice-recorder';
 
 type MessageProps = {
  role: "user" | "assistant" | "code";
@@ -75,6 +76,27 @@ const Chat = ({
  scrollToBottom();
  }, [messages]);
 
+
+ const whisperRequest = async (audioFile: Blob) => {
+ const formData = new FormData();
+ formData.append('file', audioFile, 'audio.wav');
+ console.log("got audio file," , audioFile)
+ try {
+ const response = await fetch('/api/whisper', {
+ method: 'POST',
+ body: formData,
+ });
+ const { text, error } = await response.json();
+ if (response.ok) {
+ console.log('text:', text);
+ } else {
+ console.log('Error:', error);
+ }
+ } catch (error) {
+ console.log('Error:', error);
+ }
+ };
+
  // create a new threadID when chat component created
  useEffect(() => {
  const createThread = async () => {
@@ -297,6 +319,7 @@ const Chat = ({
  >
  Send
  </button>
+ <AudioRecorder onRecordingComplete={(audioBlob) => whisperRequest(audioBlob)}/>
  </form>
  </div>
  );

app/components/control-pad.tsx

@@ -6,7 +6,7 @@ import styles from './control-pad.module.css';
 const ControlPad = () => {
  const sendCommand = async (command: string) => {
  try {
- const response = await fetch(`http:https://192.168.86.48:5000/${command}`, {
+ const response = await fetch(`${process.env.NEXT_PUBLIC_SERVER_URL}/${command}`, {
  method: 'POST',
  headers: {
  'Content-Type': 'application/json',

app/utils/car-control.ts

@@ -1,5 +1,6 @@
 async function handleMoveForward(args) {
- const response = await fetch("http:https://192.168.86.48:5000/move_forward", {
+ console.log("url is ", process.env.NEXT_PUBLIC_SERVER_URL)
+ const response = await fetch(`${process.env.NEXT_PUBLIC_SERVER_URL}/move_forward`, {
  method: "POST",
  headers: {
  "Content-Type": "application/json",
@@ -10,7 +11,7 @@ async function handleMoveForward(args) {
  }
 
  async function handleMoveBackward(args) {
- const response = await fetch("http:https://192.168.86.48:5000/move_backward", {
+ const response = await fetch(`${process.env.NEXT_PUBLIC_SERVER_URL}/move_backward`, {
  method: "POST",
  headers: {
  "Content-Type": "application/json",
@@ -21,7 +22,7 @@ async function handleMoveForward(args) {
  }
 
  async function handleTurnLeft(args) {
- const response = await fetch("http:https://192.168.86.48:5000/turn_left", {
+ const response = await fetch(`${process.env.NEXT_PUBLIC_SERVER_URL}/turn_left`, {
  method: "POST",
  headers: {
  "Content-Type": "application/json",
@@ -32,7 +33,7 @@ async function handleMoveForward(args) {
  }
 
  async function handleTurnRight(args) {
- const response = await fetch("http:https://192.168.86.48:5000/turn_right", {
+ const response = await fetch(`${process.env.NEXT_PUBLIC_SERVER_URL}/turn_right`, {
  method: "POST",
  headers: {
  "Content-Type": "application/json",
@@ -43,7 +44,7 @@ async function handleMoveForward(args) {
  }
 
  async function handleStop() {
- const response = await fetch("http:https://192.168.86.48:5000/stop", {
+ const response = await fetch(`${process.env.NEXT_PUBLIC_SERVER_URL}/stop`, {
  method: "POST",
  headers: {
  "Content-Type": "application/json",