Backend Vision Update

backend/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.9-slim
+
+# Install ffmpeg and other necessary packages
+RUN apt-get update && \
+    apt-get install -y ffmpeg libgl1-mesa-glx libglib2.0-0 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+EXPOSE 8000
+
+CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"]

backend/README.md

@@ -0,0 +1,110 @@
+# AI Voice Services Backend
+
+This directory contains the FastAPI backend for the AI Voice Agent, facilitating communication between the M5GO device, Terp AI, and ElevenLabs.
+
+## Setup Instructions
+
+### Prerequisites
+1. **Python 3.9+** is recommended.
+2. **FFmpeg** must be installed on the system to handle audio format conversions (MP3 to 16-bit 16kHz PCM).
+   - On Ubuntu/Debian: `sudo apt install ffmpeg`
+   - On macOS: `brew install ffmpeg`
+   - On Windows: Download from the [FFmpeg website](https://ffmpeg.org/download.html) and add to PATH.
+
+### Installation
+
+1. Navigate to the `ai_services` directory.
+2. (Optional but recommended) Create a virtual environment:
+   ```bash
+   python -m venv venv
+   source venv/bin/activate  # On Windows: venv\Scripts\activate
+   ```
+3. Install the required Python packages:
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+### Configuration
+
+Update the `.env` file in this directory with your ElevenLabs credentials:
+
+```ini
+ELEVENLABS_API_KEY=your_elevenlabs_api_key_here
+ELEVENLABS_VOICE_ID=JBFqnCBsd6RMkjVDRZzb
+```
+
+## Running the Server
+
+Start the FastAPI application using Uvicorn:
+
+```bash
+uvicorn server:app --host 0.0.0.0 --port 8000
+```
+This will start the server and make it accessible on your local network on port 8000.
+
+## WebSocket Endpoints
+
+### `/ws/voice`
+
+This is the primary WebSocket endpoint used by the M5GO device for real-time voice communication.
+
+**Protocol Flow:**
+
+1. **Connection:** The client establishes a WebSocket connection to `ws://<server_ip>:8000/ws/voice`.
+2. **Streaming Audio (Client -> Server):** While the user holds the record button, the client continuously sends binary frames containing raw audio data.
+   - **Expected Format:** 16-bit signed integer, 16 kHz, Mono PCM.
+3. **End of Audio Signal (Client -> Server):** When the user releases the button, the client sends a JSON text frame to signal the end of the recording:
+   ```json
+   {
+       "event": "stop_listening"
+   }
+   ```
+4. **Processing (Server):** Upon receiving the `stop_listening` event, the server executes the AI pipeline:
+   - Transcribes the accumulated PCM audio using `faster-whisper`.
+   - Sends the transcribed text to the Terp AI conversational endpoint and waits for the full response.
+   - Sends the Terp AI response text to ElevenLabs TTS.
+   - Converts the received TTS audio to 16-bit 16kHz Mono PCM.
+5. **Streaming Response (Server -> Client):** The server sends the converted PCM audio back to the client as binary frames.
+6. **End of Response (Server -> Client):** The server sends an empty binary frame (`b""`) to signal that playback is complete.
+
+## REST Endpoints
+
+### `/api/vision/room-status` (POST)
+
+This endpoint uses a YOLO object detection model to detect people and chairs in a room image, determining if the room is full and pairing the closest chairs to people.
+
+**Request:**
+- `file`: (Required) The image file to analyze (e.g., JPEG, PNG) sent as multipart form-data.
+
+**Response:**
+Returns a JSON object detailing the room status, counts, and pairings.
+
+```json
+{
+  "room_status": "full",
+  "counts": {
+    "people": 2,
+    "chairs": 2
+  },
+  "pairs": [
+    {
+      "person_index": 0,
+      "chair_index": 1,
+      "distance": 150.5
+    }
+  ],
+  "details": {
+    "people": [ ... ],
+    "chairs": [ ... ]
+  }
+}
+```
+
+## Client Integration Notes
+
+For the ESP32/M5GO client (`m5go/main.py`), ensure you update the `WS_URL` variable to point to the correct local IP address of the machine running this backend server.
+
+```python
+# In m5go/main.py
+WS_URL = "ws://192.168.1.100:8000/ws/voice" 
+```

backend/requirements.txt

@@ -0,0 +1,10 @@
+fastapi
+uvicorn
+websockets
+faster-whisper
+requests
+python-dotenv
+python-multipart
+ultralytics
+opencv-python-headless
+scipy

backend/server.py

@@ -0,0 +1,255 @@
+import os
+import io
+import struct
+import tempfile
+import subprocess
+import requests
+import json
+import base64
+import urllib.request
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect, File, UploadFile
+from dotenv import load_dotenv
+from vision import analyze_room_image
+
+load_dotenv()
+
+app = FastAPI()
+
+SAMPLE_RATE = 16000
+BITS_PER_SAMPLE = 16
+NUM_CHANNELS = 1
+
+CONVERSATION_ID = os.getenv("TERP_AI_CONVERSATION_ID", "5e752e56-06c6-ec73-1f13-456029ce1299")
+HEADERS = {
+    "accept": "*/*",
+    "accept-language": "en-US,en;q=0.9",
+    "authorization": f"Bearer {os.getenv('TERP_AI_BEARER_TOKEN', '')}",
+    "content-type": "application/json",
+    "origin": "https://patriotai.gmu.edu",
+    "referer": f"https://patriotai.gmu.edu/chat/8c3fc7f0-7c8b-4f2f-849c-5e2a45915066/{CONVERSATION_ID}",
+    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
+    "x-timezone": "America/New_York",
+}
+
+def _write_wav_to_buffer(pcm_data: bytes) -> bytes:
+    """Wrap raw PCM data in a WAV header and return the full WAV bytes."""
+    data_size = len(pcm_data)
+    byte_rate = SAMPLE_RATE * NUM_CHANNELS * (BITS_PER_SAMPLE // 8)
+    block_align = NUM_CHANNELS * (BITS_PER_SAMPLE // 8)
+
+    buf = io.BytesIO()
+    buf.write(b"RIFF")
+    buf.write(struct.pack("<I", 36 + data_size))
+    buf.write(b"WAVE")
+    buf.write(b"fmt ")
+    buf.write(struct.pack("<I", 16))
+    buf.write(struct.pack("<H", 1))  # PCM
+    buf.write(struct.pack("<H", NUM_CHANNELS))
+    buf.write(struct.pack("<I", SAMPLE_RATE))
+    buf.write(struct.pack("<I", byte_rate))
+    buf.write(struct.pack("<H", block_align))
+    buf.write(struct.pack("<H", BITS_PER_SAMPLE))
+    buf.write(b"data")
+    buf.write(struct.pack("<I", data_size))
+    buf.write(pcm_data)
+    return buf.getvalue()
+
+def _transcribe_pcm(pcm_data: bytes) -> str:
+    """Transcribe raw PCM audio using faster-whisper via a temp WAV file."""
+    from faster_whisper import WhisperModel
+    wav_data = _write_wav_to_buffer(pcm_data)
+
+    tmp_fd, tmp_path = tempfile.mkstemp(suffix=".wav")
+    try:
+        with os.fdopen(tmp_fd, "wb") as f:
+            f.write(wav_data)
+
+        # Initialize the model (using base model for speed)
+        model = WhisperModel("base", device="cpu", compute_type="int8")
+        segments, _ = model.transcribe(tmp_path, beam_size=5)
+        text = " ".join([segment.text for segment in segments])
+        return text.strip()
+    finally:
+        if os.path.exists(tmp_path):
+            os.remove(tmp_path)
+
+def get_terp_ai_response(message: str) -> str:
+    """Send text to Terp AI and return the full response."""
+    url = f"https://patriotai.gmu.edu/api/internal/userConversations/{CONVERSATION_ID}/segments"
+    data = json.dumps({
+        "question": message,
+        "visionImageIds": [],
+        "attachmentIds": [],
+        "segmentTraceLogLevel": "NonPersisted"
+    }).encode("utf-8")
+
+    req = urllib.request.Request(url, data=data, method="POST")
+    for key, value in HEADERS.items():
+        req.add_header(key, value)
+
+    full_response = ""
+    event = None
+    try:
+        with urllib.request.urlopen(req) as response:
+            while True:
+                line = response.readline()
+                if not line:
+                    break
+                line = line.decode("utf-8").strip()
+                if line.startswith("event: "):
+                    event = line[7:]
+                elif line.startswith("data: "):
+                    data = line[6:]
+                    decoded = base64.b64decode(data).decode("utf-8")
+                    if event == "response-updated":
+                        full_response += decoded
+    except Exception as e:
+        print(f"Terp AI error: {e}")
+        return "I am sorry, there was an error connecting to Terp AI."
+    
+    return full_response
+
+def _convert_to_pcm(audio_data: bytes, input_format: str = "mp3") -> bytes | None:
+    """Convert audio data to 16-bit 16 kHz mono PCM using ffmpeg."""
+    try:
+        result = subprocess.run(
+            [
+                "ffmpeg", "-y",
+                "-f", input_format, "-i", "pipe:0",
+                "-f", "s16le",
+                "-acodec", "pcm_s16le",
+                "-ar", str(SAMPLE_RATE),
+                "-ac", str(NUM_CHANNELS),
+                "pipe:1",
+            ],
+            input=audio_data,
+            capture_output=True,
+            timeout=15,
+        )
+
+        if result.returncode != 0:
+            print(f"ffmpeg conversion failed: {result.stderr.decode()[:200]}")
+            return None
+
+        return result.stdout
+
+    except FileNotFoundError:
+        print("ffmpeg not installed")
+        return None
+    except subprocess.TimeoutExpired:
+        print("ffmpeg conversion timed out")
+        return None
+
+def _generate_tts(text: str) -> bytes | None:
+    """Generate speech audio from text using ElevenLabs TTS API."""
+    api_key = os.getenv("ELEVENLABS_API_KEY")
+    voice_id = os.getenv("ELEVENLABS_VOICE_ID", "JBFqnCBsd6RMkjVDRZzb")
+    if not api_key or api_key == "your_elevenlabs_api_key_here":
+        print("ElevenLabs API key not configured")
+        return None
+
+    url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"
+
+    headers = {
+        "xi-api-key": api_key,
+        "Content-Type": "application/json",
+        "Accept": "audio/mpeg",
+    }
+
+    payload = {
+        "text": text,
+        "model_id": "eleven_flash_v2_5",
+        "voice_settings": {
+            "stability": 0.5,
+            "similarity_boost": 0.75,
+            "style": 0.0,
+            "use_speaker_boost": True,
+        },
+    }
+
+    try:
+        resp = requests.post(url, json=payload, headers=headers, timeout=30)
+        resp.raise_for_status()
+        mp3_data = resp.content
+
+        if not mp3_data:
+            return None
+
+        # Convert MP3 to 16-bit 16 kHz mono PCM
+        return _convert_to_pcm(mp3_data, input_format="mp3")
+
+    except requests.exceptions.RequestException as e:
+        print(f"ElevenLabs TTS error: {e}")
+        return None
+
+@app.websocket("/ws/voice")
+async def websocket_voice(websocket: WebSocket):
+    await websocket.accept()
+    print("Device connected to WebSocket.")
+    audio_buffer = bytearray()
+    
+    try:
+        while True:
+            data = await websocket.receive()
+            
+            if "bytes" in data:
+                audio_buffer.extend(data["bytes"])
+                
+            elif "text" in data:
+                try:
+                    msg = json.loads(data["text"])
+                    if msg.get("event") == "stop_listening":
+                        pcm_data = bytes(audio_buffer)
+                        audio_buffer = bytearray()  # Reset for next time
+                        
+                        print(f"Received stop_listening event. Buffer size: {len(pcm_data)} bytes.")
+                        
+                        if len(pcm_data) < 3200:
+                            print("Audio too short, ignoring.")
+                            await websocket.send_bytes(b"")
+                            continue
+                            
+                        # Step 1: Speech to Text
+                        print("Transcribing...")
+                        user_text = _transcribe_pcm(pcm_data)
+                        if not user_text:
+                            print("Transcription failed or empty.")
+                            await websocket.send_bytes(b"")
+                            continue
+                        
+                        print(f"User said: {user_text}")
+                        
+                        # Step 2: Terp AI
+                        print("Sending to Terp AI...")
+                        ai_response_text = get_terp_ai_response(user_text)
+                        if not ai_response_text:
+                            print("No response from Terp AI.")
+                            await websocket.send_bytes(b"")
+                            continue
+                        
+                        print(f"Terp AI response: {ai_response_text}")
+                        
+                        # Step 3: Text to Speech
+                        print("Generating TTS...")
+                        tts_pcm = _generate_tts(ai_response_text)
+                        
+                        if tts_pcm:
+                            print(f"Sending {len(tts_pcm)} bytes of PCM back to device.")
+                            await websocket.send_bytes(tts_pcm)
+                        else:
+                            print("TTS failed.")
+                            await websocket.send_bytes(b"")
+                            
+                except json.JSONDecodeError:
+                    pass
+                except Exception as e:
+                    print(f"Error processing message: {e}")
+                    await websocket.send_bytes(b"")
+    except WebSocketDisconnect:
+        print("Device disconnected.")
+
+@app.post("/api/vision/room-status")
+async def check_room_status(file: UploadFile = File(...)):
+    contents = await file.read()
+    result = analyze_room_image(contents)
+    return result

backend/vision.py

@@ -0,0 +1,102 @@
+import cv2
+import numpy as np
+from ultralytics import YOLO
+from scipy.spatial.distance import cdist
+from scipy.optimize import linear_sum_assignment
+
+# Load YOLOv8 nano model (downloads automatically if not found)
+# 'yolov8n.pt' is lightweight and fast for this purpose
+try:
+    model = YOLO("yolov8n.pt")
+except Exception as e:
+    print(f"Error loading YOLO model: {e}")
+    model = None
+
+# COCO Class IDs
+PERSON_CLASS_ID = 0
+CHAIR_CLASS_ID = 56
+
+def analyze_room_image(image_bytes: bytes):
+    if not model:
+        return {"error": "Vision model is not loaded"}
+
+    # Convert bytes to numpy array then to cv2 image
+    nparr = np.frombuffer(image_bytes, np.uint8)
+    img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+    
+    if img is None:
+        return {"error": "Invalid image format"}
+
+    # Run inference
+    results = model(img)
+    
+    people = []
+    chairs = []
+
+    for result in results:
+        boxes = result.boxes
+        for box in boxes:
+            cls_id = int(box.cls[0])
+            conf = float(box.conf[0])
+            
+            # Extract center of bounding box
+            x1, y1, x2, y2 = box.xyxy[0]
+            cx = (x1 + x2) / 2.0
+            cy = (y1 + y2) / 2.0
+            centroid = [float(cx), float(cy)]
+
+            # Only consider detections with confidence > 0.3
+            if conf > 0.3:
+                if cls_id == PERSON_CLASS_ID:
+                    people.append({
+                        "centroid": centroid, 
+                        "box": [float(x1), float(y1), float(x2), float(y2)], 
+                        "conf": conf
+                    })
+                elif cls_id == CHAIR_CLASS_ID:
+                    chairs.append({
+                        "centroid": centroid, 
+                        "box": [float(x1), float(y1), float(x2), float(y2)], 
+                        "conf": conf
+                    })
+
+    num_people = len(people)
+    num_chairs = len(chairs)
+
+    pairs = []
+    
+    # Bipartite matching if both people and chairs exist
+    if num_people > 0 and num_chairs > 0:
+        people_coords = [p["centroid"] for p in people]
+        chairs_coords = [c["centroid"] for c in chairs]
+        
+        # Distance matrix (Euclidean distances)
+        dist_matrix = cdist(people_coords, chairs_coords, metric='euclidean')
+        
+        # Hungarian algorithm to minimize total distance for pairings
+        row_ind, col_ind = linear_sum_assignment(dist_matrix)
+        
+        for person_idx, chair_idx in zip(row_ind, col_ind):
+            distance = float(dist_matrix[person_idx, chair_idx])
+            pairs.append({
+                "person_index": int(person_idx),
+                "chair_index": int(chair_idx),
+                "distance": distance
+            })
+
+    # Basic logic: room is full if there are at least as many people as chairs. 
+    # Can be adjusted based on specific room definitions
+    is_full = num_people >= num_chairs if num_chairs > 0 else False
+
+    return {
+        "room_status": "full" if is_full else "available",
+        "counts": {
+            "people": num_people,
+            "chairs": num_chairs
+        },
+        "pairs": pairs,
+        "details": {
+            "people": people,
+            "chairs": chairs
+        }
+    }