import os import io import struct import tempfile import subprocess import asyncio import requests import json import base64 import urllib.request import time import sys # Add scripts directory to path to import fake data generator sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts')) try: from generate_fake_data import get_fake_data except ImportError: print("Warning: Could not import get_fake_data from scripts/generate_fake_data.py") def get_fake_data(locations): return [] from fastapi import FastAPI, WebSocket, WebSocketDisconnect, File, UploadFile from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse, Response from pydantic import BaseModel from typing import List, Optional from datetime import datetime, timedelta from pymongo import MongoClient from dotenv import load_dotenv from vision import analyze_room_image from google import genai from fastapi.middleware.cors import CORSMiddleware load_dotenv() app = FastAPI() app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) USE_DB = os.getenv("USE_DB", "false").lower() == "true" import certifi if USE_DB: MONGO_URI = os.getenv("MONGODB_URI", "mongodb://localhost:27017/") mongo_client = MongoClient(MONGO_URI, tlsCAFile=certifi.where()) db = mongo_client.study_buddy_db study_rooms_collection = db.study_rooms else: print("Running in in-memory mode. MongoDB is disabled. Set USE_DB=true to enable.") # Fake data cache _fake_data_cache = None _fake_data_cache_time = 0 def _get_cached_fake_data(): global _fake_data_cache, _fake_data_cache_time now = time.time() # Cache for 5 minutes (300 seconds) if _fake_data_cache is None or now - _fake_data_cache_time > 300: # Assuming UMD_LOCATIONS is defined further down, but we can just use the global _fake_data_cache = get_fake_data(UMD_LOCATIONS) _fake_data_cache_time = now return _fake_data_cache class GeoJSONPoint(BaseModel): type: str = "Point" coordinates: List[float] class StudyRoomData(BaseModel): room_id: Optional[str] = None location: GeoJSONPoint db: float date: Optional[datetime] = None SAMPLE_RATE = 16000 BITS_PER_SAMPLE = 16 NUM_CHANNELS = 1 CONVERSATION_ID = os.getenv("TERP_AI_CONVERSATION_ID", "37fa27cc-542a-c8a8-9c31-9d1954fdc1d2") HEADERS = { "accept": "*/*", "accept-language": "en-US,en;q=0.9,de-DE;q=0.8,de;q=0.7", "authorization": f"Bearer {os.getenv('TERP_AI_BEARER_TOKEN', '')}", "baggage": "sentry-environment=TerpAI,sentry-release=2.2605.4472,sentry-public_key=c41f6dfb98d5bed12037e17e78c2c5d3,sentry-trace_id=250c82a03041415b99422d838ccc7003,sentry-org_id=4504359075840000,sentry-sampled=false,sentry-sample_rand=0.34017479518051186,sentry-sample_rate=0", "content-type": "application/json", "origin": "https://terpai.umd.edu", "priority": "u=1, i", "referer": f"https://terpai.umd.edu/chat/1eaa95ea-9b73-4850-8534-d1552401513a/{CONVERSATION_ID}", "sec-ch-ua": "\"Chromium\";v=\"146\", \"Not-A.Brand\";v=\"24\", \"Microsoft Edge\";v=\"146\"", "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": "\"Windows\"", "sec-fetch-dest": "empty", "sec-fetch-mode": "cors", "sec-fetch-site": "same-origin", "sentry-trace": "250c82a03041415b99422d838ccc7003-9a9ec11d7fd0293b-0", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0", "x-cosmos-session-281286": "0:-1", "x-cosmos-session-295334": "0:-1", "x-cosmos-session-317755": "0:-1", "x-cosmos-session-382299": "0:-1", "x-cosmos-session-418988": "0:-1", "x-cosmos-session-793952": "0:-1", "x-request-id": "6a128b8a-7f63-4f97-a40b-bfd31b4a376e", "x-timezone": "America/New_York", } def _write_wav_to_buffer(pcm_data: bytes, sample_rate: int = SAMPLE_RATE) -> bytes: """Wrap raw PCM data in a WAV header and return the full WAV bytes.""" data_size = len(pcm_data) byte_rate = sample_rate * NUM_CHANNELS * (BITS_PER_SAMPLE // 8) block_align = NUM_CHANNELS * (BITS_PER_SAMPLE // 8) buf = io.BytesIO() buf.write(b"RIFF") buf.write(struct.pack(" str: """Transcribe raw PCM audio using faster-whisper via a temp WAV file.""" from faster_whisper import WhisperModel print(f" Using sample rate: {sample_rate} Hz") num_samples = len(pcm_data) // 2 if num_samples > 0: samples = list(struct.unpack(f"<{num_samples}h", pcm_data[:num_samples * 2])) min_s, max_s = min(samples), max(samples) mean_s = sum(samples) / num_samples rms = (sum(s * s for s in samples) / num_samples) ** 0.5 print(f" PCM stats (raw): {num_samples} samples, min={min_s}, max={max_s}, mean={mean_s:.1f}, RMS={rms:.1f}") dc_offset = int(round(mean_s)) samples = [max(-32768, min(32767, s - dc_offset)) for s in samples] pcm_data = struct.pack(f"<{num_samples}h", *samples) rms_fixed = (sum(s * s for s in samples) / num_samples) ** 0.5 print(f" PCM stats (fixed): DC offset removed={dc_offset}, RMS={rms_fixed:.1f}") wav_data = _write_wav_to_buffer(pcm_data, sample_rate=sample_rate) tmp_fd, tmp_path = tempfile.mkstemp(suffix=".wav") try: with os.fdopen(tmp_fd, "wb") as f: f.write(wav_data) debug_path = os.path.join(os.path.dirname(__file__), "debug_audio.wav") with open(debug_path, "wb") as df: df.write(wav_data) print(f" Debug WAV saved to: {debug_path}") model = WhisperModel("base", device="cpu", compute_type="int8") segments, info = model.transcribe(tmp_path, beam_size=5) seg_list = list(segments) print(f" Whisper: {len(seg_list)} segments, language={info.language}, prob={info.language_probability:.2f}") for i, seg in enumerate(seg_list): print(f" Seg {i}: [{seg.start:.1f}s-{seg.end:.1f}s] '{seg.text}'") text = " ".join([seg.text for seg in seg_list]) return text.strip() finally: if os.path.exists(tmp_path): os.remove(tmp_path) def get_terp_ai_response(message: str) -> str: """Send text to Terp AI and return the full response. Fallback to Gemini if needed.""" url = f"https://terpai.umd.edu/api/internal/userConversations/{CONVERSATION_ID}/segments" payload = { "question": message, "visionImageIds": [], "attachmentIds": [], "segmentTraceLogLevel": "NonPersisted", "lineage": { "parentSegmentId": "83f997ca-5089-4568-ae23-fb2d5a6d5855", "lineageType": "Question" } } full_response = "" event = None try: resp = requests.post(url, json=payload, headers=HEADERS, stream=True, timeout=10, verify=False) resp.raise_for_status() for line in resp.iter_lines(decode_unicode=True): if not line: continue if line.startswith("event: "): event = line[7:] elif line.startswith("data: "): data = line[6:] decoded = base64.b64decode(data).decode("utf-8") if event == "response-updated": full_response += decoded resp.close() if full_response: return full_response else: raise Exception("Empty response from Terp AI") except Exception as e: print(f"Terp AI error, falling back to Gemini: {e}") try: api_key = os.getenv("GEMINI_API_KEY") if not api_key or api_key == "your_gemini_api_key": return "Terp AI is unavailable and Gemini fallback is not configured." client = genai.Client(api_key=api_key) response = client.models.generate_content( model='gemini-2.5-flash', contents=message ) return response.text except Exception as gemini_e: print(f"Gemini fallback error: {gemini_e}") return "I am sorry, both Terp AI and the Gemini fallback encountered an error." def _convert_to_pcm(audio_data: bytes, input_format: str = "mp3") -> Optional[bytes]: """Convert audio data to 16-bit 16 kHz mono PCM using ffmpeg.""" try: result = subprocess.run( [ "ffmpeg", "-y", "-f", input_format, "-i", "pipe:0", "-f", "s16le", "-acodec", "pcm_s16le", "-ar", str(SAMPLE_RATE), "-ac", str(NUM_CHANNELS), "pipe:1", ], input=audio_data, capture_output=True, timeout=15, ) if result.returncode != 0: print(f"ffmpeg conversion failed: {result.stderr.decode()[:200]}") return None return result.stdout except FileNotFoundError: print("ffmpeg not installed") return None except subprocess.TimeoutExpired: print("ffmpeg conversion timed out") return None def _generate_tts(text: str) -> Optional[bytes]: """Generate speech audio from text using ElevenLabs TTS API.""" api_key = os.getenv("ELEVENLABS_API_KEY") voice_id = os.getenv("ELEVENLABS_VOICE_ID", "JBFqnCBsd6RMkjVDRZzb") if not api_key or api_key == "your_elevenlabs_api_key_here": print("ElevenLabs API key not configured") return None url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}?output_format=pcm_16000" headers = { "xi-api-key": api_key, "Content-Type": "application/json", "Accept": "application/octet-stream", } payload = { "text": text, "model_id": "eleven_flash_v2_5", "voice_settings": { "stability": 0.5, "similarity_boost": 0.75, "style": 0.0, "use_speaker_boost": True, }, } try: resp = requests.post(url, json=payload, headers=headers, timeout=30) resp.raise_for_status() pcm_data = resp.content if not pcm_data: return None print(f"TTS: received {len(pcm_data)} bytes of PCM audio") return pcm_data except requests.exceptions.RequestException as e: print(f"ElevenLabs TTS error: {e}") return None _latest_tts_wav = None @app.get("/api/tts-audio") async def get_tts_audio(): global _latest_tts_wav if _latest_tts_wav is None: return Response(status_code=404, content=b"No audio available") return Response(content=_latest_tts_wav, media_type="audio/wav") @app.websocket("/ws/voice") async def websocket_voice(websocket: WebSocket): await websocket.accept() print("Device connected to WebSocket.") audio_buffer = bytearray() try: while True: data = await websocket.receive() if "bytes" in data: audio_buffer.extend(data["bytes"]) elif "text" in data: try: msg = json.loads(data["text"]) if msg.get("event") == "stop_listening": pcm_data = bytes(audio_buffer) audio_buffer = bytearray() device_sample_rate = msg.get("sample_rate", SAMPLE_RATE) print(f"Received stop_listening event. Buffer size: {len(pcm_data)} bytes, sample_rate: {device_sample_rate} Hz") if len(pcm_data) < 3200: print("Audio too short, ignoring.") await websocket.send_bytes(b"") continue print("Transcribing...") user_text = _transcribe_pcm(pcm_data, sample_rate=device_sample_rate) if not user_text: print("Transcription failed or empty.") await websocket.send_text(json.dumps({"event": "error", "msg": "No speech detected"})) continue print(f"User said: {user_text}") print("Sending to Terp AI...") context_str = get_latest_locations_context() augmented_prompt = f"USER ASKS: {user_text}\n\n[SYSTEM CONTEXT - LATEST UMD ROOM STATS TO HELP YOU ANSWER IF ASKED]:\n{context_str}" ai_response_text = get_terp_ai_response(augmented_prompt) if not ai_response_text: print("No response from Terp AI.") await websocket.send_text(json.dumps({"event": "error", "msg": "No AI response"})) continue print(f"Terp AI response: {ai_response_text}") print("Generating TTS...") tts_pcm = _generate_tts(ai_response_text) if tts_pcm: global _latest_tts_wav _latest_tts_wav = _write_wav_to_buffer(tts_pcm) print(f"TTS WAV ready: {len(_latest_tts_wav)} bytes, serving via /api/tts-audio") await websocket.send_text(json.dumps({ "event": "tts_ready", "size": len(_latest_tts_wav) })) else: print("TTS failed.") await websocket.send_text(json.dumps({"event": "error", "msg": "TTS failed"})) except json.JSONDecodeError: pass except Exception as e: print(f"Error processing message: {e}") await websocket.send_text(json.dumps({"event": "error", "msg": str(e)[:100]})) except (WebSocketDisconnect, RuntimeError): print("Device disconnected.") @app.post("/api/vision/room-status") async def check_room_status(file: UploadFile = File(...)): contents = await file.read() result = analyze_room_image(contents) return result UMD_LOCATIONS = [ { "id": 'esj', "name": 'Edward St. John (ESJ)', "lng": -76.94209511596014, "lat": 38.987133359608755 }, { "id": 'mckeldin', "name": 'McKeldin Library', "lng": -76.94494907523277, "lat": 38.986021017749366 }, { "id": 'hornbake', "name": 'Hornbake Library', "lng": -76.94161787005467, "lat": 38.988233373664826 }, { "id": 'stem', "name": 'STEM Library', "lng": -76.93942003731279, "lat": 38.988991437126195 }, { "id": 'clarice', "name": 'Clarice Library', "lng": -76.9500912552473, "lat": 38.990547823732285 }, { "id": 'yahentamitsi', "name": 'Yahentamitsi', "lng": -76.9448027183373, "lat": 38.99108961575231 }, { "id": 'iribe', "name": 'Iribe', "lng": -76.93643838603555, "lat": 38.98933701397555 }, { "id": 'reckord', "name": 'Reckord Armory', "lng": -76.93897470250619, "lat": 38.98609556181066 }, { "id": 'stamp', "name": 'Stamp Student Union', "lng": -76.94473083972326, "lat": 38.988130238874874 } ] def get_latest_locations_context() -> str: """Fetch the latest stats for each known location to feed as AI context.""" room_dict = {loc["id"]: loc["name"] for loc in UMD_LOCATIONS} if USE_DB: twenty_four_hours_ago = datetime.utcnow() - timedelta(hours=24) pipeline = [ {"$match": {"date": {"$gte": twenty_four_hours_ago}}}, {"$sort": {"date": -1}}, {"$group": { "_id": "$room_id", "latest_db": {"$first": "$db"}, "time": {"$first": "$date"} }} ] latest_stats = list(study_rooms_collection.aggregate(pipeline)) else: fake_data = _get_cached_fake_data() latest_stats_map = {} # Data is naturally sorted chronologically in our generator, so reverse it for d in reversed(fake_data): if d["room_id"] not in latest_stats_map: latest_stats_map[d["room_id"]] = { "_id": d["room_id"], "latest_db": d["db"], "time": d["date"] } latest_stats = list(latest_stats_map.values()) if not latest_stats: return "No recent location noise stats available today." lines = ["Latest Study Room Stats:"] for stat in latest_stats: room_id = stat.get("_id") name = room_dict.get(room_id, room_id) db = stat.get("latest_db", 0.0) status = "Quiet" if isinstance(db, (int, float)): if db >= 65: status = "Loud" elif db >= 55: status = "Moderate" lines.append(f"- {name}: Noise Level {db:.1f} dB ({status})") return "\n".join(lines) @app.post("/api/study-rooms") async def create_study_room_data(data: StudyRoomData): is_valid_location = False req_lng, req_lat = data.location.coordinates[0], data.location.coordinates[1] for loc in UMD_LOCATIONS: if abs(loc["lng"] - req_lng) < 0.0001 and abs(loc["lat"] - req_lat) < 0.0001: is_valid_location = True data.location.coordinates = [loc["lng"], loc["lat"]] data.room_id = loc["id"] break if not is_valid_location: from fastapi import HTTPException raise HTTPException(status_code=400, detail="Invalid location. Coordinates must correspond to a known UMD location.") if not data.date: data.date = datetime.utcnow() doc = data.dict() if USE_DB: result = study_rooms_collection.insert_one(doc) return {"id": str(result.inserted_id), "room_id": data.room_id, "status": "success"} else: return {"id": "dummy_id", "room_id": data.room_id, "status": "success (in-memory, not saved)"} @app.get("/api/study-rooms") async def get_study_room_data(): rooms = list(study_rooms_collection.find({}, {"_id": 0})) return {"data": rooms} @app.get("/api/study-rooms/history") async def get_study_room_history(): """Get all study room data from the last 24 hours.""" if USE_DB: twenty_four_hours_ago = datetime.utcnow() - timedelta(hours=24) rooms = list(study_rooms_collection.find( {"date": {"$gte": twenty_four_hours_ago}}, {"_id": 0} ).sort("date", -1)) else: rooms = _get_cached_fake_data() # Ensure we don't leak ObjectIds or non-serializable stuff # Dates are naturally sorted but let's reverse them to match MongoDB behavior (newest first) rooms = list(reversed(rooms)) return {"data": rooms} @app.get("/{full_path:path}") async def serve_spa(full_path: str): static_dir = "static" if not os.path.exists(static_dir): return {"error": "Static directory not found. Please build the frontend."} static_path = os.path.join(static_dir, full_path) if os.path.isfile(static_path): return FileResponse(static_path) index_path = os.path.join(static_dir, "index.html") if os.path.exists(index_path): return FileResponse(index_path) return {"error": "index.html not found in static directory"}