Hackathons/HushMap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

M5 Update and Web Voice Agent using Terp AI.

Browse Source
This commit is contained in:
SirBlob
2026-04-12 08:33:28 -04:00
parent 88de831198
commit e6c7ed230b
12 changed files with 1038 additions and 524 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/README.md
+20 -9
View File
@@ -26,11 +26,14 @@ This directory contains the FastAPI backend for the AI Voice Agent, facilitating
### Configuration
Update the `.env` file in this directory with your ElevenLabs credentials:
Update the `.env` file in this directory with your credentials:
```ini
ELEVENLABS_API_KEY=your_elevenlabs_api_key_here
ELEVENLABS_API_KEY=sk_...
ELEVENLABS_VOICE_ID=JBFqnCBsd6RMkjVDRZzb
TERP_AI_BEARER_TOKEN=eyJhbGciOiJSUz...
TERP_AI_CONVERSATION_ID=37fa27cc-542a-c8a8-9c31-9d1954fdc1d2
MONGODB_URI=mongodb+srv://...
```
## Running the Server
@@ -60,12 +63,18 @@ This is the primary WebSocket endpoint used by the M5GO device for real-time voi
}
```
4. **Processing (Server):** Upon receiving the `stop_listening` event, the server executes the AI pipeline:
- Transcribes the accumulated PCM audio using `faster-whisper`.
- Sends the transcribed text to the Terp AI conversational endpoint and waits for the full response.
- Sends the Terp AI response text to ElevenLabs TTS.
- Converts the received TTS audio to 16-bit 16kHz Mono PCM.
5. **Streaming Response (Server -> Client):** The server sends the converted PCM audio back to the client as binary frames.
6. **End of Response (Server -> Client):** The server sends an empty binary frame (`b""`) to signal that playback is complete.
- Transcribes the accumulated Int16 PCM audio organically using `faster-whisper`.
- Injects a MongoDB aggregate map of the latest 24hr Campus Location noise levels seamlessly into the LLM system prompt.
- Sends the transcribed text & location context to the Terp AI conversational endpoint and waits for the full response.
- Streams the Terp AI response text directly to ElevenLabs TTS and demands `pcm_16000` via URL flags natively!
5. **TTS Endpoint Notification**: The server saves the TTS audio buffer and pushes a JSON:
```json
{
"event": "tts_ready",
"size": 105000
}
```
6. **Audio Callback**: Client queries `GET /api/tts-audio` to play the binary wav response.
## REST Endpoints
@@ -126,7 +135,9 @@ Returns a list of all recorded study room data from the last 24 hours, sorted by
## Client Integration Notes
For the ESP32/M5GO client (`m5go/main.py`), ensure you update the `WS_URL` variable to point to the correct local IP address of the machine running this backend server.
For the ESP32/M5GO hardware client (`m5go/main.py`), ensure you update the `WS_URL` variable to point to the correct internal server IP.
For the Web Frontend (`VoiceButton.svelte`), it uses standard Web Audio API's `ScriptProcessorNode` to bridge the Float32 arrays strictly into 16-Bit Mono over a dynamic WebSocket tunnel automatically.
```python
# In m5go/main.py
backend/server.py
+154 -50
View File
@@ -3,13 +3,17 @@ import io
import struct
import tempfile
import subprocess
import asyncio
import requests
import json
import base64
import urllib.request
# import ssl
# ssl._create_default_https_context = ssl._create_unverified_context
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, File, UploadFile
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
from fastapi.responses import FileResponse, Response
from pydantic import BaseModel
from typing import List, Optional
from datetime import datetime, timedelta
@@ -53,22 +57,38 @@ SAMPLE_RATE = 16000
BITS_PER_SAMPLE = 16
NUM_CHANNELS = 1
CONVERSATION_ID = os.getenv("TERP_AI_CONVERSATION_ID", "5e752e56-06c6-ec73-1f13-456029ce1299")
CONVERSATION_ID = os.getenv("TERP_AI_CONVERSATION_ID", "37fa27cc-542a-c8a8-9c31-9d1954fdc1d2")
HEADERS = {
"accept": "*/*",
"accept-language": "en-US,en;q=0.9",
"accept-language": "en-US,en;q=0.9,de-DE;q=0.8,de;q=0.7",
"authorization": f"Bearer {os.getenv('TERP_AI_BEARER_TOKEN', '')}",
"baggage": "sentry-environment=TerpAI,sentry-release=2.2605.4472,sentry-public_key=c41f6dfb98d5bed12037e17e78c2c5d3,sentry-trace_id=250c82a03041415b99422d838ccc7003,sentry-org_id=4504359075840000,sentry-sampled=false,sentry-sample_rand=0.34017479518051186,sentry-sample_rate=0",
"content-type": "application/json",
"origin": "https://patriotai.gmu.edu",
"referer": f"https://patriotai.gmu.edu/chat/8c3fc7f0-7c8b-4f2f-849c-5e2a45915066/{CONVERSATION_ID}",
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
"origin": "https://terpai.umd.edu",
"priority": "u=1, i",
"referer": f"https://terpai.umd.edu/chat/1eaa95ea-9b73-4850-8534-d1552401513a/{CONVERSATION_ID}",
"sec-ch-ua": "\"Chromium\";v=\"146\", \"Not-A.Brand\";v=\"24\", \"Microsoft Edge\";v=\"146\"",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "\"Windows\"",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"sentry-trace": "250c82a03041415b99422d838ccc7003-9a9ec11d7fd0293b-0",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0",
"x-cosmos-session-281286": "0:-1#10931",
"x-cosmos-session-295334": "0:-1#749854",
"x-cosmos-session-317755": "0:-1#191601",
"x-cosmos-session-382299": "0:-1#265024",
"x-cosmos-session-418988": "0:-1#4058856",
"x-cosmos-session-793952": "0:-1#14004",
"x-request-id": "6a128b8a-7f63-4f97-a40b-bfd31b4a376e",
"x-timezone": "America/New_York",
}
def _write_wav_to_buffer(pcm_data: bytes) -> bytes:
def _write_wav_to_buffer(pcm_data: bytes, sample_rate: int = SAMPLE_RATE) -> bytes:
"""Wrap raw PCM data in a WAV header and return the full WAV bytes."""
data_size = len(pcm_data)
byte_rate = SAMPLE_RATE * NUM_CHANNELS * (BITS_PER_SAMPLE // 8)
byte_rate = sample_rate * NUM_CHANNELS * (BITS_PER_SAMPLE // 8)
block_align = NUM_CHANNELS * (BITS_PER_SAMPLE // 8)
buf = io.BytesIO()
@@ -79,7 +99,7 @@ def _write_wav_to_buffer(pcm_data: bytes) -> bytes:
buf.write(struct.pack("<I", 16))
buf.write(struct.pack("<H", 1)) # PCM
buf.write(struct.pack("<H", NUM_CHANNELS))
buf.write(struct.pack("<I", SAMPLE_RATE))
buf.write(struct.pack("<I", sample_rate))
buf.write(struct.pack("<I", byte_rate))
buf.write(struct.pack("<H", block_align))
buf.write(struct.pack("<H", BITS_PER_SAMPLE))
@@ -88,20 +108,50 @@ def _write_wav_to_buffer(pcm_data: bytes) -> bytes:
buf.write(pcm_data)
return buf.getvalue()
def _transcribe_pcm(pcm_data: bytes) -> str:
def _transcribe_pcm(pcm_data: bytes, sample_rate: int = SAMPLE_RATE) -> str:
"""Transcribe raw PCM audio using faster-whisper via a temp WAV file."""
from faster_whisper import WhisperModel
wav_data = _write_wav_to_buffer(pcm_data)
print(f" Using sample rate: {sample_rate} Hz")
# Debug: analyze PCM audio quality
num_samples = len(pcm_data) // 2
if num_samples > 0:
samples = list(struct.unpack(f"<{num_samples}h", pcm_data[:num_samples * 2]))
min_s, max_s = min(samples), max(samples)
mean_s = sum(samples) / num_samples
rms = (sum(s * s for s in samples) / num_samples) ** 0.5
print(f" PCM stats (raw): {num_samples} samples, min={min_s}, max={max_s}, mean={mean_s:.1f}, RMS={rms:.1f}")
# Remove DC offset (center audio at 0)
dc_offset = int(round(mean_s))
samples = [max(-32768, min(32767, s - dc_offset)) for s in samples]
pcm_data = struct.pack(f"<{num_samples}h", *samples)
# Stats after correction
rms_fixed = (sum(s * s for s in samples) / num_samples) ** 0.5
print(f" PCM stats (fixed): DC offset removed={dc_offset}, RMS={rms_fixed:.1f}")
wav_data = _write_wav_to_buffer(pcm_data, sample_rate=sample_rate)
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".wav")
try:
with os.fdopen(tmp_fd, "wb") as f:
f.write(wav_data)
# Save a debug copy so we can listen
debug_path = os.path.join(os.path.dirname(__file__), "debug_audio.wav")
with open(debug_path, "wb") as df:
df.write(wav_data)
print(f" Debug WAV saved to: {debug_path}")
# Initialize the model (using base model for speed)
model = WhisperModel("base", device="cpu", compute_type="int8")
segments, _ = model.transcribe(tmp_path, beam_size=5)
text = " ".join([segment.text for segment in segments])
segments, info = model.transcribe(tmp_path, beam_size=5)
seg_list = list(segments)
print(f" Whisper: {len(seg_list)} segments, language={info.language}, prob={info.language_probability:.2f}")
for i, seg in enumerate(seg_list):
print(f" Seg {i}: [{seg.start:.1f}s-{seg.end:.1f}s] '{seg.text}'")
text = " ".join([seg.text for seg in seg_list])
return text.strip()
finally:
if os.path.exists(tmp_path):
@@ -109,34 +159,34 @@ def _transcribe_pcm(pcm_data: bytes) -> str:
def get_terp_ai_response(message: str) -> str:
"""Send text to Terp AI and return the full response."""
url = f"https://patriotai.gmu.edu/api/internal/userConversations/{CONVERSATION_ID}/segments"
data = json.dumps({
url = f"https://terpai.umd.edu/api/internal/userConversations/{CONVERSATION_ID}/segments"
payload = {
"question": message,
"visionImageIds": [],
"attachmentIds": [],
"segmentTraceLogLevel": "NonPersisted"
}).encode("utf-8")
req = urllib.request.Request(url, data=data, method="POST")
for key, value in HEADERS.items():
req.add_header(key, value)
"segmentTraceLogLevel": "NonPersisted",
"lineage": {
"parentSegmentId": "83f997ca-5089-4568-ae23-fb2d5a6d5855",
"lineageType": "Question"
}
}
full_response = ""
event = None
try:
with urllib.request.urlopen(req) as response:
while True:
line = response.readline()
if not line:
break
line = line.decode("utf-8").strip()
if line.startswith("event: "):
event = line[7:]
elif line.startswith("data: "):
data = line[6:]
decoded = base64.b64decode(data).decode("utf-8")
if event == "response-updated":
full_response += decoded
resp = requests.post(url, json=payload, headers=HEADERS, stream=True, timeout=30, verify=False)
resp.raise_for_status()
for line in resp.iter_lines(decode_unicode=True):
if not line:
continue
if line.startswith("event: "):
event = line[7:]
elif line.startswith("data: "):
data = line[6:]
decoded = base64.b64decode(data).decode("utf-8")
if event == "response-updated":
full_response += decoded
resp.close()
except Exception as e:
print(f"Terp AI error: {e}")
return "I am sorry, there was an error connecting to Terp AI."
@@ -182,12 +232,13 @@ def _generate_tts(text: str) -> bytes | None:
print("ElevenLabs API key not configured")
return None
url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"
# Request PCM directly — no ffmpeg needed
url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}?output_format=pcm_16000"
headers = {
"xi-api-key": api_key,
"Content-Type": "application/json",
"Accept": "audio/mpeg",
"Accept": "application/octet-stream",
}
payload = {
@@ -204,18 +255,28 @@ def _generate_tts(text: str) -> bytes | None:
try:
resp = requests.post(url, json=payload, headers=headers, timeout=30)
resp.raise_for_status()
mp3_data = resp.content
pcm_data = resp.content
if not mp3_data:
if not pcm_data:
return None
# Convert MP3 to 16-bit 16 kHz mono PCM
return _convert_to_pcm(mp3_data, input_format="mp3")
print(f"TTS: received {len(pcm_data)} bytes of PCM audio")
return pcm_data
except requests.exceptions.RequestException as e:
print(f"ElevenLabs TTS error: {e}")
return None
# Latest TTS WAV stored in memory for HTTP download by M5GO
_latest_tts_wav = None
@app.get("/api/tts-audio")
async def get_tts_audio():
global _latest_tts_wav
if _latest_tts_wav is None:
return Response(status_code=404, content=b"No audio available")
return Response(content=_latest_tts_wav, media_type="audio/wav")
@app.websocket("/ws/voice")
async def websocket_voice(websocket: WebSocket):
await websocket.accept()
@@ -235,8 +296,9 @@ async def websocket_voice(websocket: WebSocket):
if msg.get("event") == "stop_listening":
pcm_data = bytes(audio_buffer)
audio_buffer = bytearray() # Reset for next time
device_sample_rate = msg.get("sample_rate", SAMPLE_RATE)
print(f"Received stop_listening event. Buffer size: {len(pcm_data)} bytes.")
print(f"Received stop_listening event. Buffer size: {len(pcm_data)} bytes, sample_rate: {device_sample_rate} Hz")
if len(pcm_data) < 3200:
print("Audio too short, ignoring.")
@@ -245,20 +307,22 @@ async def websocket_voice(websocket: WebSocket):
# Step 1: Speech to Text
print("Transcribing...")
user_text = _transcribe_pcm(pcm_data)
user_text = _transcribe_pcm(pcm_data, sample_rate=device_sample_rate)
if not user_text:
print("Transcription failed or empty.")
await websocket.send_bytes(b"")
await websocket.send_text(json.dumps({"event": "error", "msg": "No speech detected"}))
continue
print(f"User said: {user_text}")
# Step 2: Terp AI
print("Sending to Terp AI...")
ai_response_text = get_terp_ai_response(user_text)
context_str = get_latest_locations_context()
augmented_prompt = f"USER ASKS: {user_text}\n\n[SYSTEM CONTEXT - LATEST UMD ROOM STATS TO HELP YOU ANSWER IF ASKED]:\n{context_str}"
ai_response_text = get_terp_ai_response(augmented_prompt)
if not ai_response_text:
print("No response from Terp AI.")
await websocket.send_bytes(b"")
await websocket.send_text(json.dumps({"event": "error", "msg": "No AI response"}))
continue
print(f"Terp AI response: {ai_response_text}")
@@ -268,18 +332,24 @@ async def websocket_voice(websocket: WebSocket):
tts_pcm = _generate_tts(ai_response_text)
if tts_pcm:
print(f"Sending {len(tts_pcm)} bytes of PCM back to device.")
await websocket.send_bytes(tts_pcm)
# Save as WAV for HTTP download by M5GO
global _latest_tts_wav
_latest_tts_wav = _write_wav_to_buffer(tts_pcm)
print(f"TTS WAV ready: {len(_latest_tts_wav)} bytes, serving via /api/tts-audio")
await websocket.send_text(json.dumps({
"event": "tts_ready",
"size": len(_latest_tts_wav)
}))
else:
print("TTS failed.")
await websocket.send_bytes(b"")
await websocket.send_text(json.dumps({"event": "error", "msg": "TTS failed"}))
except json.JSONDecodeError:
pass
except Exception as e:
print(f"Error processing message: {e}")
await websocket.send_bytes(b"")
except WebSocketDisconnect:
await websocket.send_text(json.dumps({"event": "error", "msg": str(e)[:100]}))
except (WebSocketDisconnect, RuntimeError):
print("Device disconnected.")
@app.post("/api/vision/room-status")
@@ -300,6 +370,40 @@ UMD_LOCATIONS = [
{ "id": 'stamp', "name": 'Stamp Student Union', "lng": -76.94473083972326, "lat": 38.988130238874874 }
]
def get_latest_locations_context() -> str:
"""Fetch the latest stats for each known location to feed as AI context."""
twenty_four_hours_ago = datetime.utcnow() - timedelta(hours=24)
pipeline = [
{"$match": {"date": {"$gte": twenty_four_hours_ago}}},
{"$sort": {"date": -1}},
{"$group": {
"_id": "$room_id",
"latest_db": {"$first": "$db"},
"time": {"$first": "$date"}
}}
]
latest_stats = list(study_rooms_collection.aggregate(pipeline))
if not latest_stats:
return "No recent location noise stats available today."
room_dict = {loc["id"]: loc["name"] for loc in UMD_LOCATIONS}
lines = ["Latest Study Room Stats:"]
for stat in latest_stats:
room_id = stat.get("_id")
name = room_dict.get(room_id, room_id)
db = stat.get("latest_db", 0.0)
status = "Quiet"
if isinstance(db, (int, float)):
if db >= 65: status = "Loud"
elif db >= 55: status = "Moderate"
lines.append(f"- {name}: Noise Level {db:.1f} dB ({status})")
return "\n".join(lines)
@app.post("/api/study-rooms")
async def create_study_room_data(data: StudyRoomData):
# Check if the coordinates match one of the known locations (with small tolerance)