diff --git a/README.md b/README.md index e193104..32c73ea 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# BitCamp 2026 - AI Study Buddy & Room Monitor +# HUSHMAP - AI Study Buddy & Room Monitor This project is a comprehensive solution featuring an M5GO smart device integration, an AI Voice and Vision Backend, and a Svelte frontend dashboard. It connects physical hardware to advanced AI models (Terp AI, ElevenLabs, YOLOv8) to provide a real-time study buddy experience and a study room occupancy monitor. @@ -18,7 +18,8 @@ bun run dev --open ### 2. AI Backend Services (`/backend`) A FastAPI backend providing two core capabilities: -- **Real-time Voice WebSockets (`/ws/voice`)**: Connects the M5GO device to STT (faster-whisper), an LLM (Terp AI), and TTS (ElevenLabs). It streams audio bytes natively over WebSockets. +- **Real-time Voice WebSockets (`/ws/voice`)**: Connects the M5GO device AND the web dashboard to STT (faster-whisper), LLMs (Terp AI), and TTS (ElevenLabs). It streams 16-bit PCM audio bytes natively over WebSockets in full-duplex. +- **Context DB Aggregation**: TerpAI automatically queries the MongoDB `study_rooms_collection` to gather live hardware decibel readings globally before answering your prompt. - **Vision Occupancy API (`/api/vision/room-status`)**: Uses YOLOv8 object detection to identify people and chairs in a room image, determining if a study room is fully occupied and pairing the closest person to an available chair. **Developing:** @@ -51,11 +52,13 @@ Make sure you set up your `.env` variables before running the Docker containers Create a `.env` in the `/backend` folder: ```ini -ELEVENLABS_API_KEY=your_elevenlabs_api_key_here +ELEVENLABS_API_KEY=sk_... ELEVENLABS_VOICE_ID=JBFqnCBsd6RMkjVDRZzb -TERP_AI_BEARER_TOKEN=your_jwt_token_here -TERP_AI_CONVERSATION_ID=5e752e56-06c6-ec73-1f13-456029ce1299 -MONGODB_URI=mongodb_url_here +TERP_AI_BEARER_TOKEN=eyJhbGciOiJ... +TERP_AI_CONVERSATION_ID=37fa27cc-542a-c8a8-9c31-9d1954fdc1d2 +MONGODB_URI=mongodb+srv://... ``` +Update your `.env` to match the exact `authorization: Bearer` and `parentSegmentId` context from TerpAI if timeouts occur. + Update the `/m5go/main.py` file to include your Wi-Fi credentials and the correct local IP for the WebSocket (`WS_URL`). \ No newline at end of file diff --git a/backend/README.md b/backend/README.md index 9062ceb..1237b5b 100644 --- a/backend/README.md +++ b/backend/README.md @@ -26,11 +26,14 @@ This directory contains the FastAPI backend for the AI Voice Agent, facilitating ### Configuration -Update the `.env` file in this directory with your ElevenLabs credentials: +Update the `.env` file in this directory with your credentials: ```ini -ELEVENLABS_API_KEY=your_elevenlabs_api_key_here +ELEVENLABS_API_KEY=sk_... ELEVENLABS_VOICE_ID=JBFqnCBsd6RMkjVDRZzb +TERP_AI_BEARER_TOKEN=eyJhbGciOiJSUz... +TERP_AI_CONVERSATION_ID=37fa27cc-542a-c8a8-9c31-9d1954fdc1d2 +MONGODB_URI=mongodb+srv://... ``` ## Running the Server @@ -60,12 +63,18 @@ This is the primary WebSocket endpoint used by the M5GO device for real-time voi } ``` 4. **Processing (Server):** Upon receiving the `stop_listening` event, the server executes the AI pipeline: - - Transcribes the accumulated PCM audio using `faster-whisper`. - - Sends the transcribed text to the Terp AI conversational endpoint and waits for the full response. - - Sends the Terp AI response text to ElevenLabs TTS. - - Converts the received TTS audio to 16-bit 16kHz Mono PCM. -5. **Streaming Response (Server -> Client):** The server sends the converted PCM audio back to the client as binary frames. -6. **End of Response (Server -> Client):** The server sends an empty binary frame (`b""`) to signal that playback is complete. + - Transcribes the accumulated Int16 PCM audio organically using `faster-whisper`. + - Injects a MongoDB aggregate map of the latest 24hr Campus Location noise levels seamlessly into the LLM system prompt. + - Sends the transcribed text & location context to the Terp AI conversational endpoint and waits for the full response. + - Streams the Terp AI response text directly to ElevenLabs TTS and demands `pcm_16000` via URL flags natively! +5. **TTS Endpoint Notification**: The server saves the TTS audio buffer and pushes a JSON: + ```json + { + "event": "tts_ready", + "size": 105000 + } + ``` +6. **Audio Callback**: Client queries `GET /api/tts-audio` to play the binary wav response. ## REST Endpoints @@ -126,7 +135,9 @@ Returns a list of all recorded study room data from the last 24 hours, sorted by ## Client Integration Notes -For the ESP32/M5GO client (`m5go/main.py`), ensure you update the `WS_URL` variable to point to the correct local IP address of the machine running this backend server. +For the ESP32/M5GO hardware client (`m5go/main.py`), ensure you update the `WS_URL` variable to point to the correct internal server IP. + +For the Web Frontend (`VoiceButton.svelte`), it uses standard Web Audio API's `ScriptProcessorNode` to bridge the Float32 arrays strictly into 16-Bit Mono over a dynamic WebSocket tunnel automatically. ```python # In m5go/main.py diff --git a/backend/server.py b/backend/server.py index d028029..aacc420 100644 --- a/backend/server.py +++ b/backend/server.py @@ -3,13 +3,17 @@ import io import struct import tempfile import subprocess +import asyncio import requests import json import base64 import urllib.request +# import ssl +# ssl._create_default_https_context = ssl._create_unverified_context + from fastapi import FastAPI, WebSocket, WebSocketDisconnect, File, UploadFile from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, Response from pydantic import BaseModel from typing import List, Optional from datetime import datetime, timedelta @@ -53,22 +57,38 @@ SAMPLE_RATE = 16000 BITS_PER_SAMPLE = 16 NUM_CHANNELS = 1 -CONVERSATION_ID = os.getenv("TERP_AI_CONVERSATION_ID", "5e752e56-06c6-ec73-1f13-456029ce1299") +CONVERSATION_ID = os.getenv("TERP_AI_CONVERSATION_ID", "37fa27cc-542a-c8a8-9c31-9d1954fdc1d2") HEADERS = { "accept": "*/*", - "accept-language": "en-US,en;q=0.9", + "accept-language": "en-US,en;q=0.9,de-DE;q=0.8,de;q=0.7", "authorization": f"Bearer {os.getenv('TERP_AI_BEARER_TOKEN', '')}", + "baggage": "sentry-environment=TerpAI,sentry-release=2.2605.4472,sentry-public_key=c41f6dfb98d5bed12037e17e78c2c5d3,sentry-trace_id=250c82a03041415b99422d838ccc7003,sentry-org_id=4504359075840000,sentry-sampled=false,sentry-sample_rand=0.34017479518051186,sentry-sample_rate=0", "content-type": "application/json", - "origin": "https://patriotai.gmu.edu", - "referer": f"https://patriotai.gmu.edu/chat/8c3fc7f0-7c8b-4f2f-849c-5e2a45915066/{CONVERSATION_ID}", - "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + "origin": "https://terpai.umd.edu", + "priority": "u=1, i", + "referer": f"https://terpai.umd.edu/chat/1eaa95ea-9b73-4850-8534-d1552401513a/{CONVERSATION_ID}", + "sec-ch-ua": "\"Chromium\";v=\"146\", \"Not-A.Brand\";v=\"24\", \"Microsoft Edge\";v=\"146\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Windows\"", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-origin", + "sentry-trace": "250c82a03041415b99422d838ccc7003-9a9ec11d7fd0293b-0", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0", + "x-cosmos-session-281286": "0:-1#10931", + "x-cosmos-session-295334": "0:-1#749854", + "x-cosmos-session-317755": "0:-1#191601", + "x-cosmos-session-382299": "0:-1#265024", + "x-cosmos-session-418988": "0:-1#4058856", + "x-cosmos-session-793952": "0:-1#14004", + "x-request-id": "6a128b8a-7f63-4f97-a40b-bfd31b4a376e", "x-timezone": "America/New_York", } -def _write_wav_to_buffer(pcm_data: bytes) -> bytes: +def _write_wav_to_buffer(pcm_data: bytes, sample_rate: int = SAMPLE_RATE) -> bytes: """Wrap raw PCM data in a WAV header and return the full WAV bytes.""" data_size = len(pcm_data) - byte_rate = SAMPLE_RATE * NUM_CHANNELS * (BITS_PER_SAMPLE // 8) + byte_rate = sample_rate * NUM_CHANNELS * (BITS_PER_SAMPLE // 8) block_align = NUM_CHANNELS * (BITS_PER_SAMPLE // 8) buf = io.BytesIO() @@ -79,7 +99,7 @@ def _write_wav_to_buffer(pcm_data: bytes) -> bytes: buf.write(struct.pack(" bytes: buf.write(pcm_data) return buf.getvalue() -def _transcribe_pcm(pcm_data: bytes) -> str: +def _transcribe_pcm(pcm_data: bytes, sample_rate: int = SAMPLE_RATE) -> str: """Transcribe raw PCM audio using faster-whisper via a temp WAV file.""" from faster_whisper import WhisperModel - wav_data = _write_wav_to_buffer(pcm_data) + print(f" Using sample rate: {sample_rate} Hz") + + # Debug: analyze PCM audio quality + num_samples = len(pcm_data) // 2 + if num_samples > 0: + samples = list(struct.unpack(f"<{num_samples}h", pcm_data[:num_samples * 2])) + min_s, max_s = min(samples), max(samples) + mean_s = sum(samples) / num_samples + rms = (sum(s * s for s in samples) / num_samples) ** 0.5 + print(f" PCM stats (raw): {num_samples} samples, min={min_s}, max={max_s}, mean={mean_s:.1f}, RMS={rms:.1f}") + + # Remove DC offset (center audio at 0) + dc_offset = int(round(mean_s)) + samples = [max(-32768, min(32767, s - dc_offset)) for s in samples] + pcm_data = struct.pack(f"<{num_samples}h", *samples) + + # Stats after correction + rms_fixed = (sum(s * s for s in samples) / num_samples) ** 0.5 + print(f" PCM stats (fixed): DC offset removed={dc_offset}, RMS={rms_fixed:.1f}") + + wav_data = _write_wav_to_buffer(pcm_data, sample_rate=sample_rate) tmp_fd, tmp_path = tempfile.mkstemp(suffix=".wav") try: with os.fdopen(tmp_fd, "wb") as f: f.write(wav_data) + + # Save a debug copy so we can listen + debug_path = os.path.join(os.path.dirname(__file__), "debug_audio.wav") + with open(debug_path, "wb") as df: + df.write(wav_data) + print(f" Debug WAV saved to: {debug_path}") # Initialize the model (using base model for speed) model = WhisperModel("base", device="cpu", compute_type="int8") - segments, _ = model.transcribe(tmp_path, beam_size=5) - text = " ".join([segment.text for segment in segments]) + segments, info = model.transcribe(tmp_path, beam_size=5) + seg_list = list(segments) + print(f" Whisper: {len(seg_list)} segments, language={info.language}, prob={info.language_probability:.2f}") + for i, seg in enumerate(seg_list): + print(f" Seg {i}: [{seg.start:.1f}s-{seg.end:.1f}s] '{seg.text}'") + text = " ".join([seg.text for seg in seg_list]) return text.strip() finally: if os.path.exists(tmp_path): @@ -109,34 +159,34 @@ def _transcribe_pcm(pcm_data: bytes) -> str: def get_terp_ai_response(message: str) -> str: """Send text to Terp AI and return the full response.""" - url = f"https://patriotai.gmu.edu/api/internal/userConversations/{CONVERSATION_ID}/segments" - data = json.dumps({ + url = f"https://terpai.umd.edu/api/internal/userConversations/{CONVERSATION_ID}/segments" + payload = { "question": message, "visionImageIds": [], "attachmentIds": [], - "segmentTraceLogLevel": "NonPersisted" - }).encode("utf-8") - - req = urllib.request.Request(url, data=data, method="POST") - for key, value in HEADERS.items(): - req.add_header(key, value) + "segmentTraceLogLevel": "NonPersisted", + "lineage": { + "parentSegmentId": "83f997ca-5089-4568-ae23-fb2d5a6d5855", + "lineageType": "Question" + } + } full_response = "" event = None try: - with urllib.request.urlopen(req) as response: - while True: - line = response.readline() - if not line: - break - line = line.decode("utf-8").strip() - if line.startswith("event: "): - event = line[7:] - elif line.startswith("data: "): - data = line[6:] - decoded = base64.b64decode(data).decode("utf-8") - if event == "response-updated": - full_response += decoded + resp = requests.post(url, json=payload, headers=HEADERS, stream=True, timeout=30, verify=False) + resp.raise_for_status() + for line in resp.iter_lines(decode_unicode=True): + if not line: + continue + if line.startswith("event: "): + event = line[7:] + elif line.startswith("data: "): + data = line[6:] + decoded = base64.b64decode(data).decode("utf-8") + if event == "response-updated": + full_response += decoded + resp.close() except Exception as e: print(f"Terp AI error: {e}") return "I am sorry, there was an error connecting to Terp AI." @@ -182,12 +232,13 @@ def _generate_tts(text: str) -> bytes | None: print("ElevenLabs API key not configured") return None - url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}" + # Request PCM directly — no ffmpeg needed + url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}?output_format=pcm_16000" headers = { "xi-api-key": api_key, "Content-Type": "application/json", - "Accept": "audio/mpeg", + "Accept": "application/octet-stream", } payload = { @@ -204,18 +255,28 @@ def _generate_tts(text: str) -> bytes | None: try: resp = requests.post(url, json=payload, headers=headers, timeout=30) resp.raise_for_status() - mp3_data = resp.content + pcm_data = resp.content - if not mp3_data: + if not pcm_data: return None - # Convert MP3 to 16-bit 16 kHz mono PCM - return _convert_to_pcm(mp3_data, input_format="mp3") + print(f"TTS: received {len(pcm_data)} bytes of PCM audio") + return pcm_data except requests.exceptions.RequestException as e: print(f"ElevenLabs TTS error: {e}") return None +# Latest TTS WAV stored in memory for HTTP download by M5GO +_latest_tts_wav = None + +@app.get("/api/tts-audio") +async def get_tts_audio(): + global _latest_tts_wav + if _latest_tts_wav is None: + return Response(status_code=404, content=b"No audio available") + return Response(content=_latest_tts_wav, media_type="audio/wav") + @app.websocket("/ws/voice") async def websocket_voice(websocket: WebSocket): await websocket.accept() @@ -235,8 +296,9 @@ async def websocket_voice(websocket: WebSocket): if msg.get("event") == "stop_listening": pcm_data = bytes(audio_buffer) audio_buffer = bytearray() # Reset for next time + device_sample_rate = msg.get("sample_rate", SAMPLE_RATE) - print(f"Received stop_listening event. Buffer size: {len(pcm_data)} bytes.") + print(f"Received stop_listening event. Buffer size: {len(pcm_data)} bytes, sample_rate: {device_sample_rate} Hz") if len(pcm_data) < 3200: print("Audio too short, ignoring.") @@ -245,20 +307,22 @@ async def websocket_voice(websocket: WebSocket): # Step 1: Speech to Text print("Transcribing...") - user_text = _transcribe_pcm(pcm_data) + user_text = _transcribe_pcm(pcm_data, sample_rate=device_sample_rate) if not user_text: print("Transcription failed or empty.") - await websocket.send_bytes(b"") + await websocket.send_text(json.dumps({"event": "error", "msg": "No speech detected"})) continue print(f"User said: {user_text}") # Step 2: Terp AI print("Sending to Terp AI...") - ai_response_text = get_terp_ai_response(user_text) + context_str = get_latest_locations_context() + augmented_prompt = f"USER ASKS: {user_text}\n\n[SYSTEM CONTEXT - LATEST UMD ROOM STATS TO HELP YOU ANSWER IF ASKED]:\n{context_str}" + ai_response_text = get_terp_ai_response(augmented_prompt) if not ai_response_text: print("No response from Terp AI.") - await websocket.send_bytes(b"") + await websocket.send_text(json.dumps({"event": "error", "msg": "No AI response"})) continue print(f"Terp AI response: {ai_response_text}") @@ -268,18 +332,24 @@ async def websocket_voice(websocket: WebSocket): tts_pcm = _generate_tts(ai_response_text) if tts_pcm: - print(f"Sending {len(tts_pcm)} bytes of PCM back to device.") - await websocket.send_bytes(tts_pcm) + # Save as WAV for HTTP download by M5GO + global _latest_tts_wav + _latest_tts_wav = _write_wav_to_buffer(tts_pcm) + print(f"TTS WAV ready: {len(_latest_tts_wav)} bytes, serving via /api/tts-audio") + await websocket.send_text(json.dumps({ + "event": "tts_ready", + "size": len(_latest_tts_wav) + })) else: print("TTS failed.") - await websocket.send_bytes(b"") + await websocket.send_text(json.dumps({"event": "error", "msg": "TTS failed"})) except json.JSONDecodeError: pass except Exception as e: print(f"Error processing message: {e}") - await websocket.send_bytes(b"") - except WebSocketDisconnect: + await websocket.send_text(json.dumps({"event": "error", "msg": str(e)[:100]})) + except (WebSocketDisconnect, RuntimeError): print("Device disconnected.") @app.post("/api/vision/room-status") @@ -300,6 +370,40 @@ UMD_LOCATIONS = [ { "id": 'stamp', "name": 'Stamp Student Union', "lng": -76.94473083972326, "lat": 38.988130238874874 } ] +def get_latest_locations_context() -> str: + """Fetch the latest stats for each known location to feed as AI context.""" + twenty_four_hours_ago = datetime.utcnow() - timedelta(hours=24) + pipeline = [ + {"$match": {"date": {"$gte": twenty_four_hours_ago}}}, + {"$sort": {"date": -1}}, + {"$group": { + "_id": "$room_id", + "latest_db": {"$first": "$db"}, + "time": {"$first": "$date"} + }} + ] + latest_stats = list(study_rooms_collection.aggregate(pipeline)) + + if not latest_stats: + return "No recent location noise stats available today." + + room_dict = {loc["id"]: loc["name"] for loc in UMD_LOCATIONS} + + lines = ["Latest Study Room Stats:"] + for stat in latest_stats: + room_id = stat.get("_id") + name = room_dict.get(room_id, room_id) + db = stat.get("latest_db", 0.0) + + status = "Quiet" + if isinstance(db, (int, float)): + if db >= 65: status = "Loud" + elif db >= 55: status = "Moderate" + + lines.append(f"- {name}: Noise Level {db:.1f} dB ({status})") + + return "\n".join(lines) + @app.post("/api/study-rooms") async def create_study_room_data(data: StudyRoomData): # Check if the coordinates match one of the known locations (with small tolerance) diff --git a/m5go/face_assets.py b/m5go/face_assets.py deleted file mode 100644 index 143d286..0000000 --- a/m5go/face_assets.py +++ /dev/null @@ -1,177 +0,0 @@ -# --- FACE ASSETS --- -C = { - '0': 0x222222, - 'Y': 0xFFFF00, - 'R': 0xFF0000, - 'W': 0xFFFFFF, - 'B': 0x000000, - 'P': 0xFF8888, - 'D': 0x555555 -} - -f_s_o = [ - "0000000000000000", - "0000000000000000", - "000WWW0000WWW000", - "00WWBW0000WWBW00", - "00WWBW0000WWBW00", - "000WWW0000WWW000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "YY000000000000YY", - "0YY0000000000YY0", - "00YY00000000YY00", - "000YYYYYYYYYY000", - "00000YYYYYY00000", - "0000000000000000", - "0000000000000000" -] - -f_s_h = [ - "0000000000000000", - "0000000000000000", - "0000000000000000", - "000WWW0000WWW000", - "00WWBW0000WWBW00", - "000WWW0000WWW000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "YY000000000000YY", - "0YY0000000000YY0", - "00YY00000000YY00", - "000YYYYYYYYYY000", - "00000YYYYYY00000", - "0000000000000000", - "0000000000000000" -] - -f_s_c = [ - "0000000000000000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "00WWWW0000WWWW00", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "YY000000000000YY", - "0YY0000000000YY0", - "00YY00000000YY00", - "000YYYYYYYYYY000", - "00000YYYYYY00000", - "0000000000000000", - "0000000000000000" -] - -f_a_o = [ - "0000000000000000", - "0DDDD000000DDDD0", - "00DDDD0000DDDD00", - "000DDDD00DDDD000", - "000WWW0000WWW000", - "00WWBB0000BBWW00", - "000WWW0000WWW000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "000000RRRR000000", - "0000RRRRRRRR0000", - "00RRRR0000RRRR00", - "0RRR00000000RRR0", - "0000000000000000", - "0000000000000000" -] - -f_a_h = [ - "0000000000000000", - "0DDDD000000DDDD0", - "00DDDD0000DDDD00", - "000DDDD00DDDD000", - "0000000000000000", - "000WWW0000WWW000", - "00WWBB0000BBWW00", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "000000RRRR000000", - "0000RRRRRRRR0000", - "00RRRR0000RRRR00", - "0RRR00000000RRR0", - "0000000000000000", - "0000000000000000" -] - -f_a_c = [ - "0000000000000000", - "0DDDD000000DDDD0", - "00DDDD0000DDDD00", - "000DDDD00DDDD000", - "0000000000000000", - "0000000000000000", - "000WW000000WW000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "000000RRRR000000", - "0000RRRRRRRR0000", - "00RRRR0000RRRR00", - "0RRR00000000RRR0", - "0000000000000000", - "0000000000000000" -] - -f_t_1 = [ - "0000000000000000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "000WWW0000WWW000", - "00WWWW0000WWWW00", - "000WWW0000WWW000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "000YYYYYYYYYY000", - "0000000000000000", - "0000000000000000", - "0000000000000000" -] - -f_t_2 = [ - "0000000000000000", - "0000000000000000", - "00DD00000000DD00", - "000DD000000DD000", - "0000000000000000", - "000WWW0000WWW000", - "00WWBW0000WWBW00", - "000WWW0000WWW000", - "0000000000000000", - "0000000000000000", - "0000000000000000", - "0000RRRRRRRR0000", - "000RR000000RR000", - "0000000000000000", - "0000000000000000", - "0000000000000000" -] - -def d_s(lcd, f, s_x, s_y, p_s): - for r in range(16): - c = 0 - while c < 16: - s_c = c - v = f[r][c] - while c < 16 and f[r][c] == v: - c += 1 - w = c - s_c - - x_p = s_x + (s_c * p_s) - y_p = s_y + (r * p_s) - - lcd.fillRect(x_p, y_p, w * p_s, p_s, C[v]) \ No newline at end of file diff --git a/m5go/main.py b/m5go/main.py index 8845739..eed4345 100644 --- a/m5go/main.py +++ b/m5go/main.py @@ -3,73 +3,277 @@ import time import machine import json import math -import _thread -import websocket +import usocket +import ubinascii +import os +import gc from m5stack import * from m5ui import * from uiflow import * +C = { '0': 0x222222, 'Y': 0xFFFF00, 'R': 0xFF0000, 'W': 0xFFFFFF, 'B': 0x000000, 'D': 0x555555 } +f_s_o = "00000000000000000000000000000000000WWW0000WWW00000WWBW0000WWBW0000WWBW0000WWBW00000WWW0000WWW000000000000000000000000000000000000000000000000000YY000000000000YY0YY0000000000YY000YY00000000YY00000YYYYYYYYYY00000000YYYYYY0000000000000000000000000000000000000" +f_s_c = "0000000000000000000000000000000000000000000000000000000000000000000000000000000000WWWW0000WWWW00000000000000000000000000000000000000000000000000YY000000000000YY0YY0000000000YY000YY00000000YY00000YYYYYYYYYY00000000YYYYYY0000000000000000000000000000000000000" +f_t_1 = "0000000000000000000000000000000000000000000000000000000000000000000WWW0000WWW00000WWWW0000WWWW00000WWW0000WWW00000000000000000000000000000000000000000000000000000000000000000000000000000000000000YYYYYYYYYY000000000000000000000000000000000000000000000000000" +f_t_2 = "0000000000000000000000000000000000DD00000000DD00000DD000000DD0000000000000000000000WWW0000WWW00000WWBW0000WWBW00000WWW0000WWW0000000000000000000000000000000000000000000000000000000RRRRRRRR0000000RR000000RR000000000000000000000000000000000000000000000000000" +f_listen = "0000000000000000000000000000000000WWWW0000WWWW0000WBBW0000WBBW0000WBBW0000WBBW0000WWWW0000WWWW0000000000000000000000000000000000000000000000000000000YYYYYY000000000YY0000YY00000000YY0000YY000000000YYYYYY00000000000000000000000000000000000000000000000000000" +f_speak = "00000000000000000000000000000000000WWW0000WWW00000WWBW0000WWBW0000WWBW0000WWBW00000WWW0000WWW0000000000000000000000000000000000000000000000000000000YYYYYYYY0000000YY000000YY000000YY000000YY0000000YYYYYYYY0000000000000000000000000000000000000000000000000000" +f_angry = "00000000000000000DDDD000000DDDD000DDDD0000DDDD00000DDDD00DDDD000000WWW0000WWW00000WWBB0000BBWW00000WWW0000WWW000000000000000000000000000000000000000000000000000000000RRRR0000000000RRRRRRRR000000RRRR0000RRRR000RRR00000000RRR000000000000000000000000000000000" + +def d_s(lcd, f, s_x, s_y, p_s): + for r in range(16): + c = 0 + i = r * 16 + while c < 16: + s_c = c + v = f[i + c] + while c < 16 and f[i + c] == v: + c += 1 + w = c - s_c + lcd.fillRect(s_x + s_c * p_s, s_y + r * p_s, w * p_s, p_s, C[v]) + setScreenColor(0x222222) -# --- CONFIGURATION --- +# ========================================== +# CONFIGURATION +# ========================================== + WIFI_SSID = "Blobby" WIFI_PASS = "73556088" -# Update this to the IP address of your backend server -WS_URL = "ws://192.168.137.1:8000/ws/voice" -# --------------------- +WS_URL = "ws://192.168.137.1:8000/ws/voice" -def draw_status(status, color): +# Location Settings for Noise Monitoring +CURRENT_ROOM_ID = "mckeldin" +CURRENT_LAT = 38.986021 +CURRENT_LNG = -76.944949 + +# Audio Settings +TARGET_SAMPLE_RATE = 8000 # Voice recording sample rate +AUDIO_CHUNK_SIZE = 2048 + +# ========================================== +# WEBSOCKET CLIENT +# ========================================== + +class WSClient: + def __init__(self, sock): + self._sock = sock + + def send(self, data): + if isinstance(data, str): + data = data.encode() + opcode = 0x1 + else: + opcode = 0x2 + length = len(data) + mask_key = os.urandom(4) + header = bytearray() + header.append(0x80 | opcode) + if length < 126: + header.append(0x80 | length) + elif length < 65536: + header.append(0x80 | 126) + header.append((length >> 8) & 0xFF) + header.append(length & 0xFF) + else: + header.append(0x80 | 127) + for i in range(7, -1, -1): + header.append((length >> (8 * i)) & 0xFF) + header.extend(mask_key) + masked = bytearray(data) + for i in range(length): + masked[i] ^= mask_key[i % 4] + self._sock.send(header + masked) + + def recv(self): + hdr = self._recv_exact(2) + if not hdr or len(hdr) < 2: + return None + opcode = hdr[0] & 0x0F + is_masked = (hdr[1] & 0x80) != 0 + length = hdr[1] & 0x7F + if length == 126: + ext = self._recv_exact(2) + length = (ext[0] << 8) | ext[1] + elif length == 127: + ext = self._recv_exact(8) + length = 0 + for b in ext: + length = (length << 8) | b + mask_key = self._recv_exact(4) if is_masked else None + payload = self._recv_exact(length) if length > 0 else b"" + if is_masked and mask_key and payload: + payload = bytearray(payload) + for i in range(len(payload)): + payload[i] ^= mask_key[i % 4] + payload = bytes(payload) + if opcode == 0x8: + return None + if opcode == 0x9: + self._send_pong(payload) + return self.recv() + if opcode == 0x1: + return payload.decode() if payload else "" + return payload + + def _send_pong(self, data): + mask_key = os.urandom(4) + length = len(data) if data else 0 + header = bytearray([0x8A, 0x80 | length]) + header.extend(mask_key) + if data: + masked = bytearray(data) + for i in range(length): + masked[i] ^= mask_key[i % 4] + self._sock.send(header + masked) + else: + self._sock.send(header) + + def _recv_exact(self, n): + buf = bytearray(n) + pos = 0 + while pos < n: + chunk = self._sock.recv(n - pos) + if not chunk: + return None + buf[pos:pos + len(chunk)] = chunk + pos += len(chunk) + return bytes(buf) + + def close(self): + try: + self._sock.send(bytearray([0x88, 0x80, 0, 0, 0, 0])) + except: + pass + try: + self._sock.close() + except: + pass + +def ws_connect(url): + if url.startswith("ws://"): + rest = url[5:] + else: + raise ValueError("Only ws:// supported") + if "/" in rest: + host_port = rest.split("/", 1)[0] + path = "/" + rest.split("/", 1)[1] + else: + host_port = rest + path = "/" + if ":" in host_port: + host = host_port.split(":")[0] + port = int(host_port.split(":")[1]) + else: + host = host_port + port = 80 + + addr = usocket.getaddrinfo(host, port)[0][-1] + sock = usocket.socket() + sock.connect(addr) + sock.settimeout(15) + + key = ubinascii.b2a_base64(os.urandom(16)).strip().decode() + req = ( + "GET %s HTTP/1.1\r\n" + "Host: %s\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Key: %s\r\n" + "Sec-WebSocket-Version: 13\r\n" + "\r\n" + ) % (path, host_port, key) + + sock.send(req.encode()) + resp = b"" + while b"\r\n\r\n" not in resp: + b = sock.recv(1) + if not b: + sock.close() + raise Exception("Closed during handshake") + resp += b + + status_line = resp.split(b"\r\n")[0] + if b"101" not in status_line: + sock.close() + raise Exception("Upgrade failed: " + status_line.decode()) + + return WSClient(sock) + +# ========================================== +# UI & UTILITY FUNCTIONS +# ========================================== + +_cur_face = None +def set_face(face): + global _cur_face + if face and face != _cur_face: + d_s(lcd, face, 80, 25, 10) + _cur_face = face + +def draw_status(status, color, face=None): lcd.fillRect(0, 220, 320, 20, 0x222222) lcd.print(status, int((320 - len(status) * 8) / 2), 220, color) + if face: + set_face(face) -lcd.print("Connecting to WiFi...", 0, 0, 0xFFFFFF) -wlan = network.WLAN(network.STA_IF) -wlan.active(True) -wlan.connect(WIFI_SSID, WIFI_PASS) +def connect_wifi(): + lcd.clear() + lcd.print("Connecting to WiFi...", 0, 0, 0xFFFFFF) + wlan = network.WLAN(network.STA_IF) + wlan.active(True) + if not wlan.isconnected(): + wlan.connect(WIFI_SSID, WIFI_PASS) + + attempts = 0 + while not wlan.isconnected() and attempts < 20: + time.sleep(0.5) + attempts += 1 + + if wlan.isconnected(): + lcd.clear() + lcd.print("WiFi Connected!", 0, 0, 0x00FF00) + lcd.print(wlan.ifconfig()[0], 0, 20, 0x00FF00) + time.sleep(1) + lcd.clear() + else: + lcd.clear() + lcd.print("WiFi Failed", 0, 0, 0xFF0000) + time.sleep(2) + lcd.clear() + return wlan.isconnected() -# Simple connection loop -attempts = 0 -while not wlan.isconnected() and attempts < 20: - time.sleep(0.5) - attempts += 1 +# ========================================== +# AUDIO I/O +# ========================================== -if wlan.isconnected(): - lcd.clear() - lcd.print("WiFi Connected!", 0, 0, 0x00FF00) - lcd.print(wlan.ifconfig()[0], 0, 20, 0x00FF00) - time.sleep(1) - lcd.clear() -else: - lcd.clear() - lcd.print("WiFi Failed", 0, 0, 0xFF0000) - time.sleep(2) - lcd.clear() - -try: - adc = machine.ADC(34) - adc.atten(machine.ADC.ATTN_11DB) -except: +def get_adc(): try: - adc = machine.ADC(machine.Pin(34)) - adc.atten(machine.ADC.ATTN_11DB) + a = machine.ADC(34) + a.atten(machine.ADC.ATTN_11DB) + return a except: - adc = None + try: + a = machine.ADC(machine.Pin(34)) + a.atten(machine.ADC.ATTN_11DB) + return a + except: + return None -def get_db(): - if not adc: return 30 +def get_db(adc_obj): + if not adc_obj: return 30 sum_v = 0 sum_sq = 0 count = 0 end_t = time.ticks_ms() + 40 while time.ticks_ms() < end_t: try: - v = adc.read() + v = adc_obj.read() sum_v += v sum_sq += v * v count += 1 except: pass - if count == 0: return 30 mean = sum_v / count @@ -79,249 +283,293 @@ def get_db(): amp = math.sqrt(variance) if amp <= 1: return 30 - # Use your specific calculation formula to properly scale to human DB - db = 20 * math.log10(amp) + 25 - return db + # Scale to human DB + return 20 * math.log10(amp) + 25 -def init_mic(): +def init_manual_spk(): try: - if hasattr(machine.I2S, "RX"): - audio_in = machine.I2S( - 0, - sck=machine.Pin(0), - ws=machine.Pin(0), - sd=machine.Pin(34), - mode=machine.I2S.RX, - bits=16, - format=machine.I2S.MONO, - rate=16000, - ibuf=4096 - ) - return audio_in - else: - # Fallback for old Micropython (e.g. M5Stack UIFlow) - mode = getattr(machine.I2S, "MODE_MASTER", 1) | getattr(machine.I2S, "MODE_RX", 2) - if hasattr(machine.I2S, "MODE_PDM"): - mode |= getattr(machine.I2S, "MODE_PDM", 0) - - cfmt = getattr(machine.I2S, "CHANNEL_FMT_ALL_LEFT", 1) - dfmt = getattr(machine.I2S, "FORMAT_I2S", 1) - - try_args = [ - ([getattr(machine.I2S, "NUM0", 0), mode, 16000, 16, cfmt, dfmt], {}), - ([getattr(machine.I2S, "NUM0", 0), mode, 16000, 16], {}), - ([getattr(machine.I2S, "NUM0", 0)], {"mode": mode, "sample_rate": 16000, "bits": 16, "channel_format": cfmt, "data_format": dfmt}), - ([getattr(machine.I2S, "NUM0", 0)], {"mode": mode, "sample_rate": 16000, "bits": 16}), - ([getattr(machine.I2S, "NUM0", 0)], {"mode": mode, "bck": 0, "ws": 0, "sd": 34, "sample_rate": 16000, "bits": 16}), - ([], {"mode": mode, "sample_rate": 16000, "bits": 16}), - ([getattr(machine.I2S, "NUM0", 0)], {"mode": mode}), - ([], {"mode": mode}), - ] - - audio_in = None - last_e = None - for args, kwargs in try_args: - try: - audio_in = machine.I2S(*args, **kwargs) - break - except Exception as e: - last_e = e - if audio_in is None: - raise Exception("Mic fallback failed: " + str(last_e)) - return audio_in - except Exception as e: - print("Mic I2S Error:", repr(e)) - lcd.print("Mic Err: " + str(e)[:20], 0, 40, 0xFF0000) - return None - -def init_spk(): - try: - if hasattr(machine.I2S, "TX"): - audio_out = machine.I2S( - 1, - sck=machine.Pin(12), - ws=machine.Pin(0), - sd=machine.Pin(25), # M5Stack Core/GO speaker is typically on Pin 25 - mode=machine.I2S.TX, - bits=16, - format=machine.I2S.MONO, - rate=16000, - ibuf=8192 - ) - return audio_out - else: - mode = getattr(machine.I2S, "MODE_MASTER", 1) | getattr(machine.I2S, "MODE_TX", 2) - if hasattr(machine.I2S, "MODE_DAC_BUILT_IN"): - mode |= machine.I2S.MODE_DAC_BUILT_IN - + # M5GO Core speaker sits on DAC 1 (Pin 25) + # Therefore we MUST strictly use DAC_BUILT_IN. Digital I2S on this pin is not supported! + if hasattr(machine.I2S, "MODE_DAC_BUILT_IN"): + mode = getattr(machine.I2S, "MODE_MASTER", 1) | getattr(machine.I2S, "MODE_TX", 2) | getattr(machine.I2S, "MODE_DAC_BUILT_IN", 0) cfmt = getattr(machine.I2S, "CHANNEL_FMT_RIGHT_LEFT", 1) dfmt = getattr(machine.I2S, "FORMAT_I2S_MSB", 1) - try_args = [ - ([getattr(machine.I2S, "NUM1", 1), mode, 16000, 16, cfmt, dfmt], {}), - ([getattr(machine.I2S, "NUM1", 1), mode, 16000, 16], {}), - ([getattr(machine.I2S, "NUM1", 1)], {"mode": mode, "sample_rate": 16000, "bits": 16, "channel_format": cfmt, "data_format": dfmt}), - ([getattr(machine.I2S, "NUM1", 1)], {"mode": mode, "sample_rate": 16000, "bits": 16}), - ([getattr(machine.I2S, "NUM1", 1)], {"mode": mode, "bck": 12, "ws": 0, "sd": 25, "sample_rate": 16000, "bits": 16}), + i2s_id = getattr(machine.I2S, "NUM0", 0) + + # Different MicroPython versions vary wildly on kwarg vs positional I2S init structure + try_sigs = [ + ([i2s_id], {"mode": mode, "rate": 16000, "bits": 16, "format": cfmt, "ibuf": 2048}), + ([i2s_id, mode, 16000, 16, cfmt, dfmt], {}), + ([i2s_id, mode, 16000, 16], {}), + ([i2s_id], {"mode": mode, "sample_rate": 16000, "bits": 16, "channel_format": cfmt, "data_format": dfmt}), + ([i2s_id], {"mode": mode, "sample_rate": 16000, "bits": 16}), ([], {"mode": mode, "sample_rate": 16000, "bits": 16}), - ([getattr(machine.I2S, "NUM1", 1)], {"mode": mode}), - ([], {"mode": mode}), + ([i2s_id], {"mode": mode, "rate": 16000, "bits": 16}) ] - audio_out = None - last_e = None - for args, kwargs in try_args: + last_err = None + for args, kwargs in try_sigs: try: - audio_out = machine.I2S(*args, **kwargs) - break + return machine.I2S(*args, **kwargs) except Exception as e: - last_e = e - if audio_out is None: - raise Exception("Spk fallback failed: " + str(last_e)) - return audio_out + last_err = e + + print("I2S Fallback err (exhausted):", last_err) + return None + else: + print("ERR: DAC_BUILT_IN missing on this firmware") + return None except Exception as e: - print("Spk I2S Error:", repr(e)) - lcd.print("Spk Err: " + str(e)[:20], 0, 60, 0xFF0000) + print("I2S Init err:", e) return None -def deinit_i2s(i2s_obj): - if i2s_obj: - try: - if hasattr(i2s_obj, 'deinit'): - i2s_obj.deinit() - except Exception as e: - print("I2S Deinit Error:", e) - -ws = None - -def connect_ws(): - global ws - try: - ws = websocket.WebSocket() - ws.connect(WS_URL) - return True - except Exception as e: - draw_status("WS Connection Error", 0xFF0000) +def stream_http_audio(url): + print("Streaming I2S direct from HTTP...") + audio_out = init_manual_spk() + if not audio_out: + print("Speaker init failed, cannot play audio via DAC") return False - -draw_status("Hold Button A to Talk", 0xFFFFFF) - -buf = bytearray(1024) - -# Noise monitoring variables -l_db_s = "" -s_db = 30.0 - -# Location settings - change this depending on where the M5GO is placed -CURRENT_ROOM_ID = "mckeldin" -CURRENT_LAT = 38.986021 -CURRENT_LNG = -76.944949 - -last_db_post_time = 0 - -while True: - # m5stack core button check - if btnA.isPressed(): - if not ws: - draw_status("Connecting...", 0xFFFF00) - if not connect_ws(): - time.sleep(1) - continue - draw_status("Listening...", 0x0000FF) - - # Read and send audio while button is held - audio_in = init_mic() - while btnA.isPressed(): - try: - if audio_in: - num_read = audio_in.readinto(buf) - if num_read and num_read > 0 and ws: - ws.send(buf[:num_read]) - except Exception as e: - pass - deinit_i2s(audio_in) - audio_in = None - - # Button released - draw_status("Thinking...", 0xFFFF00) - try: - if ws: - ws.send(json.dumps({"event": "stop_listening"})) - - # Wait for response audio - draw_status("Speaking...", 0x00FF00) - audio_out = init_spk() - while True: - resp = ws.recv() - if resp and isinstance(resp, bytes): - if len(resp) == 0: - break # End of audio transmission - - # Briefly pause face animation updates while speaker is playing - # to prevent dropping packets or stuttering - if audio_out: - audio_out.write(resp) - else: - break # Empty or non-bytes response means end - deinit_i2s(audio_out) - audio_out = None - except Exception as e: - draw_status("Error during playback", 0xFF0000) - ws = None # force reconnect next time - - draw_status("Hold Button A to Talk", 0xFFFFFF) - - # --------------------------------------------- - # NOISE MONITORING LOOP - # --------------------------------------------- - - r_db = get_db() - s_db = (s_db * 0.8) + (r_db * 0.2) - db = int(s_db) - - if db < 40: - i_c = 0x89b4fa - elif db < 55: - i_c = 0x94e2d5 - elif db < 65: - i_c = 0xf9e2af - elif db < 80: - i_c = 0xfab387 + if url.startswith("http://"): + rest = url[7:] else: - i_c = 0xf38ba8 - - lcd.fillRect(0, 0, 320, 4, i_c) - db_s = "Noise: %d dB" % db - if db_s != l_db_s: - lcd.fillRect(0, 4, 150, 15, 0x222222) - lcd.print(db_s, 5, 4, i_c) - l_db_s = db_s + raise ValueError("Only http:// supported") - # --------------------------------------------- - # POST DB DATA PERIODICALLY - # --------------------------------------------- - try: - now_ms = time.ticks_ms() - if time.ticks_diff(now_ms, last_db_post_time) > 15000: # Every 15 seconds - last_db_post_time = now_ms - try: - payload = { - "room_id": CURRENT_ROOM_ID, - "location": { - "type": "Point", - "coordinates": [CURRENT_LNG, CURRENT_LAT] - }, - "db": float(db) - } - # Convert ws url from ws://ip:port/ws/voice to http://ip:port/api/study-rooms - http_url = WS_URL.replace("ws://", "http://").replace("/ws/voice", "/api/study-rooms") - import urequests - res = urequests.post(http_url, json=payload) - res.close() - except Exception as e: - print("Failed to post db data:", e) - except Exception as main_e: - pass + if "/" in rest: + host_port = rest.split("/", 1)[0] + path = "/" + rest.split("/", 1)[1] + else: + host_port = rest + path = "/" - time.sleep(0.02) + if ":" in host_port: + host = host_port.split(":")[0] + port = int(host_port.split(":")[1]) + else: + host = host_port + port = 80 + + addr = usocket.getaddrinfo(host, port)[0][-1] + sock = usocket.socket() + sock.connect(addr) + sock.settimeout(30) + + req = "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (path, host_port) + sock.send(req.encode()) + + hdr = b"" + while b"\r\n\r\n" not in hdr: + b = sock.recv(1) + if not b: + sock.close() + return False + hdr += b + + # Read WAV header + header_left = 44 + while header_left > 0: + chunk = sock.recv(header_left) + if not chunk: break + header_left -= len(chunk) + + in_buf = bytearray(1024) + out_buf = bytearray(2048) + + while True: + n = 0 + while n < 1024: + chunk = sock.recv(1024 - n) + if not chunk: break + in_buf[n:n+len(chunk)] = chunk + n += len(chunk) + + if n == 0: break + + samples = n // 2 + for j in range(samples): + idx = j * 2 + # Read 16-bit Signed LE + s = in_buf[idx] | (in_buf[idx + 1] << 8) + if s >= 32768: s -= 65536 + + # Convert to Unsigned + Center Offset for DAC + u = (s + 32768) & 0xFFFF + u_lo = u & 0xFF + u_hi = u >> 8 + + # Map Stereo for built-in MSB + o_idx = j * 4 + out_buf[o_idx] = u_lo + out_buf[o_idx + 1] = u_hi + out_buf[o_idx + 2] = u_lo + out_buf[o_idx + 3] = u_hi + + try: + audio_out.write(out_buf[:samples * 4]) + except Exception as e: + print("Write err:", e) + break + + sock.close() + if hasattr(audio_out, 'deinit'): audio_out.deinit() + return True + +# ========================================== +# MAIN LOOP +# ========================================== + +def run_main(): + if not connect_wifi(): + return + + adc = get_adc() + ws = None + l_db_s = "" + s_db = 30.0 + last_db_post_time = time.ticks_ms() + + draw_status("Hold Button A to Talk", 0xFFFFFF, f_s_o) + + def maintain_ws(): + nonlocal ws + if not ws: + draw_status("Connecting...", 0xFFFF00, f_t_2) + try: + ws = ws_connect(WS_URL) + draw_status("Hold Button A to Talk", 0xFFFFFF, f_s_o) + except Exception as e: + print("WS Err:", e) + ws = None + return ws is not None + + while True: + gc.collect() + # --- Voice Interaction --- + if btnA.isPressed(): + if maintain_ws(): + draw_status("Listening...", 0x0000FF, f_listen) + + send_buf = bytearray(AUDIO_CHUNK_SIZE) + buf_pos = 0 + total_samples = 0 + + rec_start_us = time.ticks_us() + + # Fastest possible analog capture loop + while btnA.isPressed(): + try: + raw = adc.read() if adc else 2048 + sample = (raw - 2048) * 16 + # Fast clamp + if sample > 32767: sample = 32767 + elif sample < -32768: sample = -32768 + + send_buf[buf_pos] = sample & 0xFF + send_buf[buf_pos + 1] = (sample >> 8) & 0xFF + buf_pos += 2 + total_samples += 1 + + if buf_pos >= AUDIO_CHUNK_SIZE: + if ws: ws.send(bytes(send_buf)) + buf_pos = 0 + except Exception as e: + print("Rec Err:", e) + break + + if buf_pos > 0 and ws: + try: + ws.send(bytes(send_buf[:buf_pos])) + except: + pass + + draw_status("Thinking...", 0xFFFF00, f_t_1) + actual_rate = (total_samples * 1000000) // time.ticks_diff(time.ticks_us(), rec_start_us) + print("Captured at", actual_rate, "Hz") + + try: + ws.send(json.dumps({"event": "stop_listening", "sample_rate": actual_rate})) + + # Wait for TTS ready + resp = ws.recv() + if resp and isinstance(resp, str): + msg = json.loads(resp) + if msg.get("event") == "tts_ready": + # Start direct TCP stream immediately + draw_status("Speaking...", 0x00FF00, f_speak) + http_base = WS_URL.replace("ws://", "http://").replace("/ws/voice", "") + audio_url = http_base + "/api/tts-audio" + + try: + stream_http_audio(audio_url) + except Exception as e: + print("Stream Err:", e) + draw_status("Play Failed", 0xFF0000, f_s_c) + time.sleep(1) + + elif msg.get("event") == "error": + draw_status("Error: " + msg.get("msg", "")[:10], 0xFF0000, f_s_c) + time.sleep(2) + except Exception as e: + print("Comm Err:", e) + try: ws.close() + except: pass + ws = None + + draw_status("Hold Button A to Talk", 0xFFFFFF, f_s_o) + + # --- Noise Monitoring --- + r_db = get_db(adc) + s_db = (s_db * 0.8) + (r_db * 0.2) + db_val = int(s_db) + + i_c = 0x89b4fa if db_val < 40 else (0x94e2d5 if db_val < 55 else (0xf9e2af if db_val < 65 else (0xfab387 if db_val < 80 else 0xf38ba8))) + lcd.fillRect(0, 0, 320, 4, i_c) + db_s = "Noise: %d dB" % db_val + if db_s != l_db_s: + lcd.fillRect(0, 4, 150, 15, 0x222222) + lcd.print(db_s, 5, 4, i_c) + l_db_s = db_s + + # Periodic DB Posting + try: + now_ms = time.ticks_ms() + if time.ticks_diff(now_ms, last_db_post_time) > 15000: + last_db_post_time = now_ms + payload_str = '{"room_id":"%s","location":{"type":"Point","coordinates":[%s,%s]},"db":%s}' % (CURRENT_ROOM_ID, CURRENT_LNG, CURRENT_LAT, db_val) + http_url = WS_URL.replace("ws://", "http://").replace("/ws/voice", "/api/study-rooms") + + # Raw socket post + h_p = http_url.split("://")[1].split("/")[0] + p_th = "/" + http_url.split("://")[1].split("/", 1)[1] if "/" in http_url.split("://")[1] else "/" + h_b = h_p.split(":")[0] + p_r = int(h_p.split(":")[1]) if ":" in h_p else 80 + addr = usocket.getaddrinfo(h_b, p_r)[0][-1] + s = usocket.socket() + s.settimeout(5) + s.connect(addr) + req = "POST %s HTTP/1.0\r\nHost: %s\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s" % (p_th, h_p, len(payload_str), payload_str) + s.send(req.encode()) + s.close() + del s, req, payload_str + except: + pass + + # Idle Face Blinking logic + if db_val > 67: + set_face(f_angry) + else: + t = time.ticks_ms() % 5000 + if t < 200: + set_face(f_s_c) + else: + set_face(f_s_o) + + time.sleep(0.02) + +if __name__ == "__main__": + try: + run_main() + except Exception as e: + print("Fatal error:", e) + lcd.print("Error: " + str(e), 0, 100, 0xFF0000) diff --git a/website/package.json b/website/package.json index d515dd3..2f1ec32 100644 --- a/website/package.json +++ b/website/package.json @@ -14,14 +14,14 @@ "type": "module", "dependencies": { "@auth0/auth0-spa-js": "^2.19.0", - "maplibre-gl": "^5.0.1", - "chart.js": "^4.4.0", + "maplibre-gl": "^5.22.0", + "chart.js": "^4.5.1", "chartjs-adapter-date-fns": "^3.0.0", "date-fns": "^2.30.0" }, "devDependencies": { "@iconify/svelte": "^5.2.1", - "@sveltejs/adapter-node": "^5.2.10", + "@sveltejs/adapter-node": "^5.5.4", "@sveltejs/adapter-static": "^3.0.10", "@sveltejs/kit": "^2.57.1", "@sveltejs/vite-plugin-svelte": "^7.0.0", @@ -29,7 +29,7 @@ "@tailwindcss/typography": "^0.5.19", "@tailwindcss/vite": "^4.2.2", "@types/node": "^25.6.0", - "@vite-pwa/sveltekit": "^0.6.6", + "@vite-pwa/sveltekit": "^0.6.8", "svelte": "^5.55.3", "svelte-check": "^4.4.6", "tailwindcss": "^4.2.2", diff --git a/website/src/lib/components/AICallModal.svelte b/website/src/lib/components/AICallModal.svelte new file mode 100644 index 0000000..77abfcb --- /dev/null +++ b/website/src/lib/components/AICallModal.svelte @@ -0,0 +1,316 @@ + + +
+
+ + +

Live AI Assistant

+

+ {#if callState === 'idle'} + Disconnected + {:else if callState === 'listening'} + Listening... + {:else if callState === 'processing'} + Agent is thinking... + {:else if callState === 'speaking'} + Agent is speaking... + {/if} +

+ + +
+ + {#if callState === 'listening' || callState === 'speaking'} +
+
+ {/if} + +
+ {#if callState === 'processing'} + + + {:else} + + + + + {/if} +
+
+ + {#if errorMessage} +
+ {errorMessage} +
+ {/if} + + +
+ {#if callState === 'idle'} + + {:else if callState === 'listening'} + + {/if} + + +
+
+
+ + diff --git a/website/src/lib/components/VoiceButton.svelte b/website/src/lib/components/VoiceButton.svelte index d9aba7e..70df48f 100644 --- a/website/src/lib/components/VoiceButton.svelte +++ b/website/src/lib/components/VoiceButton.svelte @@ -1,12 +1,18 @@ -
+
@@ -17,7 +23,7 @@
+{#if isModalOpen} + +{/if} +