From 96272aedb73878f9fe505c6e2b776e32d80a3863 Mon Sep 17 00:00:00 2001 From: SirBlob Date: Sun, 12 Apr 2026 08:49:36 -0400 Subject: [PATCH] Code comments Removed Update --- README.md | 96 +++--- backend/README.md | 142 +++------ backend/emoji_data.py | 276 ++++++++++++++++++ backend/server.py | 107 +++---- backend/vision.py | 46 +-- m5go/main.py | 166 +++++------ scripts/ai.py | 20 +- scripts/generate_fake_data.py | 42 +-- website/src/app.d.ts | 12 +- website/src/lib/components/AICallModal.svelte | 61 ++-- .../src/lib/components/AlertsDropdown.svelte | 12 +- .../src/lib/components/InteractiveMap.svelte | 22 +- .../src/lib/components/LocationPopup.svelte | 16 +- website/src/lib/index.ts | 2 +- website/src/lib/states/alerts.svelte.ts | 20 +- website/src/lib/states/map.svelte.ts | 8 +- website/src/routes/+layout.svelte | 4 +- website/src/routes/history/+page.svelte | 34 +-- website/src/routes/layout.css | 2 +- website/src/routes/settings/+page.svelte | 16 +- website/svelte.config.js | 4 +- website/vite.config.ts | 2 +- 22 files changed, 677 insertions(+), 433 deletions(-) create mode 100644 backend/emoji_data.py diff --git a/README.md b/README.md index 32c73ea..800f3dc 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,80 @@ -# HUSHMAP - AI Study Buddy & Room Monitor +
+

HushMap

+

AI Intelligent Room Monitor

+
+

+ SvelteKit + FastAPI + MongoDB + M5GO + Docker +

+
-This project is a comprehensive solution featuring an M5GO smart device integration, an AI Voice and Vision Backend, and a Svelte frontend dashboard. It connects physical hardware to advanced AI models (Terp AI, ElevenLabs, YOLOv8) to provide a real-time study buddy experience and a study room occupancy monitor. +
-## Project Architecture +HushMap bridges the gap between hardware sensors and top-tier artificial intelligence pipelines (e.g. **Terp AI**, **ElevenLabs**, **YOLOv8**), delivering a seamless real-time learning assistant combined with live noise and occupancy metrics. -### 1. Website Frontend (`/website` & Root) -A SvelteKit application providing the user interface for our system. -- Powered by `sv` (Svelte CLI) and Bun. -- Configured for production deployment via Docker. +--- -**Developing:** -```bash -cd website -bun install -bun run dev --open -``` +## System Architecture + +### 1. Web Dashboard (`/website`) +A responsive, high-fidelity PWA frontend written in Svelte 5 and styled seamlessly with Catppuccin color guidelines. +* **Powered By**: SvelteKit, Vite, and Bun. +* **Features**: Live interactive map tracking, responsive UI, persistent theming, and an autonomous browser-based Voice Agent calling modal interface. +* **Setup**: + ```bash + cd website + bun install + bun run dev --open + ``` ### 2. AI Backend Services (`/backend`) -A FastAPI backend providing two core capabilities: -- **Real-time Voice WebSockets (`/ws/voice`)**: Connects the M5GO device AND the web dashboard to STT (faster-whisper), LLMs (Terp AI), and TTS (ElevenLabs). It streams 16-bit PCM audio bytes natively over WebSockets in full-duplex. -- **Context DB Aggregation**: TerpAI automatically queries the MongoDB `study_rooms_collection` to gather live hardware decibel readings globally before answering your prompt. -- **Vision Occupancy API (`/api/vision/room-status`)**: Uses YOLOv8 object detection to identify people and chairs in a room image, determining if a study room is fully occupied and pairing the closest person to an available chair. +A blazing fast asynchronous HTTP server facilitating audio chunking and sensor metrics logic over full-duplex sockets. +* **Core Capabilities**: + * **Voice Socket Pipelining**: WebSockets (`/ws/voice`) that hook incoming 16-bit PCM arrays into `faster-whisper`. + * **LLM Context Augmentation**: Seamlessly aggregates live MongoDB noise statistics (Decibel levels per location) to feed contextual history to the TerpAI engine! + * **Computer Vision Endpoint**: Exposes a `YOLOv8` tensor API (`/api/vision/room-status`) to parse webcam imagery, pinpoint seating capacities, and locate available chairs algorithmically. +* **Setup**: + ```bash + cd backend + pip install -r requirements.txt + uvicorn server:app --host 0.0.0.0 --port 8000 + ``` + > [!WARNING] + > Host devices must have `ffmpeg`, `libgl1-mesa-glx`, and `libglib2.0-0` binaries natively installed to encode audio buffers and execute OpenCV rendering. -**Developing:** -```bash -cd backend -pip install -r requirements.txt -uvicorn server:app --host 0.0.0.0 --port 8000 -``` -*(Requires `ffmpeg`, `libgl1-mesa-glx`, and `libglib2.0-0` installed on your system)* +### 3. M5GO Hardware Node (`/m5go`) +C-based MicroPython binaries tailored strictly for the IoT edge nodes traversing the physical campus. +* **Features**: Connects internally wired I2S Microphone blocks to route direct byte arrays securely out across WPA/WPA2 networks into the main API gateway using minimal payload overhead. Push-button PTT interfaces built directly into the screen chassis. -### 3. M5GO Device (`/m5go`) -MicroPython scripts for the M5Stack M5GO device. -- Uses `uwebsockets` to connect to the backend. -- High-quality audio I2S configuration for the internal microphone and speaker. -- Push-to-talk integration: Hold Button A to talk to the AI, release to get an audio response back. +--- -## Docker Setup +## Docker Production Setup -The entire stack can be run via Docker Compose, which builds both the Svelte website and the Python AI Backend into a single seamless container. +The entire monolithic architecture cleanly orchestrates via docker compose. Frontends compile out via SSR, and Python APIs wire natively within a segregated container network loop. ```bash docker-compose up --build ``` +> The global deployment interface listens on port `8000`. -- **App (Frontend + Backend)**: Runs on port `8000` +--- -## Configuration +## Core Configuration -Make sure you set up your `.env` variables before running the Docker containers or local servers. +Before starting services, strictly adhere to configuring your environment files (`.env`) within `/backend`: -Create a `.env` in the `/backend` folder: ```ini ELEVENLABS_API_KEY=sk_... ELEVENLABS_VOICE_ID=JBFqnCBsd6RMkjVDRZzb -TERP_AI_BEARER_TOKEN=eyJhbGciOiJ... -TERP_AI_CONVERSATION_ID=37fa27cc-542a-c8a8-9c31-9d1954fdc1d2 -MONGODB_URI=mongodb+srv://... +TERP_AI_BEARER_TOKEN=eyJhbGciOiJSUz... +TERP_AI_CONVERSATION_ID=3a150d8e-bb12-... +MONGODB_URI=mongodb+srv://user:pass@cluster0... ``` -Update your `.env` to match the exact `authorization: Bearer` and `parentSegmentId` context from TerpAI if timeouts occur. +> [!TIP] +> Ensure the `.env` mirrors your authentic `x-cosmos-session` headers and bearer tokens exported from a live browser session to prevent immediate 401 Unauthorized timeouts in the Terp AI pipeline. -Update the `/m5go/main.py` file to include your Wi-Fi credentials and the correct local IP for the WebSocket (`WS_URL`). \ No newline at end of file +For IoT clients, update `/m5go/main.py` explicitly to broadcast to your running router IP namespace matching your specific VLAN. diff --git a/backend/README.md b/backend/README.md index 1237b5b..0ef4464 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,93 +1,74 @@ -# AI Voice Services Backend +
+

HushMap: AI Services API

+

+ FastAPI + Python + YOLOv8 Vision + Whisper +

+

The central nervous system linking physical M5GO devices, external Computer Vision tensors, and Conversational NLP APIs synchronously.

+
-This directory contains the FastAPI backend for the AI Voice Agent, facilitating communication between the M5GO device, Terp AI, and ElevenLabs. +--- ## Setup Instructions ### Prerequisites -1. **Python 3.9+** is recommended. -2. **FFmpeg** must be installed on the system to handle audio format conversions (MP3 to 16-bit 16kHz PCM). - - On Ubuntu/Debian: `sudo apt install ffmpeg` - - On macOS: `brew install ffmpeg` - - On Windows: Download from the [FFmpeg website](https://ffmpeg.org/download.html) and add to PATH. +1. **Python 3.9+** is strictly recommended to support asynchronous typing paradigms. +2. **FFmpeg** must be successfully registered onto your OS PATH environments. This engine handles the core conversions decoding MP3 output arrays into 16-bit, 16kHz Mono arrays natively required for browser contexts: + - **Ubuntu/Debian**: `sudo apt install ffmpeg` + - **macOS**: `brew install ffmpeg` + - **Windows**: Install globally via the [FFmpeg website](https://ffmpeg.org/download.html). -### Installation +### Environment Initialization -1. Navigate to the `ai_services` directory. -2. (Optional but recommended) Create a virtual environment: - ```bash - python -m venv venv - source venv/bin/activate # On Windows: venv\Scripts\activate - ``` -3. Install the required Python packages: - ```bash - pip install -r requirements.txt - ``` +Bootstrap the virtual environment and initialize project dependencies: -### Configuration +```bash +cd backend +python -m venv venv +source venv/bin/activate # Windows: .\venv\Scripts\activate +pip install -r requirements.txt +``` -Update the `.env` file in this directory with your credentials: +### Configuration Tokens + +Provide runtime keys securely targeting TerpAI context queues and ElevenLabs synthesized avatars within a `.env` dotfile: ```ini ELEVENLABS_API_KEY=sk_... ELEVENLABS_VOICE_ID=JBFqnCBsd6RMkjVDRZzb TERP_AI_BEARER_TOKEN=eyJhbGciOiJSUz... -TERP_AI_CONVERSATION_ID=37fa27cc-542a-c8a8-9c31-9d1954fdc1d2 +TERP_AI_CONVERSATION_ID=37fa27cc-... MONGODB_URI=mongodb+srv://... ``` -## Running the Server - -Start the FastAPI application using Uvicorn: +To invoke the engine, simply execute Uvicorn across your `0.0.0.0` loopback: ```bash uvicorn server:app --host 0.0.0.0 --port 8000 ``` -This will start the server and make it accessible on your local network on port 8000. -## WebSocket Endpoints +--- -### `/ws/voice` +## Gateway Pipelines -This is the primary WebSocket endpoint used by the M5GO device for real-time voice communication. +### Full-Duplex Subroutines (`/ws/voice`) -**Protocol Flow:** +This WebSocket proxy establishes a fully integrated multi-turn communication bridge seamlessly interacting between Edge node Hardware APIs (ESP32/M5GO/Browsers) and NLP architectures. -1. **Connection:** The client establishes a WebSocket connection to `ws://:8000/ws/voice`. -2. **Streaming Audio (Client -> Server):** While the user holds the record button, the client continuously sends binary frames containing raw audio data. - - **Expected Format:** 16-bit signed integer, 16 kHz, Mono PCM. -3. **End of Audio Signal (Client -> Server):** When the user releases the button, the client sends a JSON text frame to signal the end of the recording: - ```json - { - "event": "stop_listening" - } - ``` -4. **Processing (Server):** Upon receiving the `stop_listening` event, the server executes the AI pipeline: - - Transcribes the accumulated Int16 PCM audio organically using `faster-whisper`. - - Injects a MongoDB aggregate map of the latest 24hr Campus Location noise levels seamlessly into the LLM system prompt. - - Sends the transcribed text & location context to the Terp AI conversational endpoint and waits for the full response. - - Streams the Terp AI response text directly to ElevenLabs TTS and demands `pcm_16000` via URL flags natively! -5. **TTS Endpoint Notification**: The server saves the TTS audio buffer and pushes a JSON: - ```json - { - "event": "tts_ready", - "size": 105000 - } - ``` -6. **Audio Callback**: Client queries `GET /api/tts-audio` to play the binary wav response. +1. **Int16 Byte Array Exchange**: Devices connect to `ws://:8000/ws/voice` and push raw binary frames asynchronously over the socket. +2. **Contextual Augmentation**: The server waits for the `"stop_listening"` payload event to signify a completed audio snippet. That float array is cast through `faster-whisper` and combined seamlessly with real-time `MongoDB` decibel tracking telemetry parameters natively attached into the `TerpAI` user conversation chunk. +3. **TTS Pipeline Rendering**: Output predictions are caught instantly, forwarded natively into the `ElevenLabs` TTS interface rendering `pcm_16000` wav codecs, and alerted back down to clients using a `tts_ready` dispatcher. -## REST Endpoints +### Tensor Vision Endpoints (`/api/vision/room-status`) -### `/api/vision/room-status` (POST) +Leveraging OpenCV bindings layered beneath a YOLOv8-driven bounding box topology detector, this `POST` API analyzes raw camera image buffers returning capacity logic natively. -This endpoint uses a YOLO object detection model to detect people and chairs in a room image, determining if the room is full and pairing the closest chairs to people. - -**Request:** -- `file`: (Required) The image file to analyze (e.g., JPEG, PNG) sent as multipart form-data. - -**Response:** -Returns a JSON object detailing the room status, counts, and pairings. +> [!NOTE] +> This API calculates euclidean distances algorithmically detecting adjacent proximities between "person" classifiers and untaken "chair" bounding frames to accurately diagnose available seats inside crowded architectures! +**Response Output Protocol:** ```json { "room_status": "full", @@ -101,45 +82,16 @@ Returns a JSON object detailing the room status, counts, and pairings. "chair_index": 1, "distance": 150.5 } - ], - "details": { - "people": [ ... ], - "chairs": [ ... ] - } -} -``` - -### `/api/study-rooms` (GET) - -Returns a list of all recorded study room data. - -### `/api/study-rooms/history` (GET) - -Returns a list of all recorded study room data from the last 24 hours, sorted by most recent first. - -**Response:** -```json -{ - "data": [ - { - "location": { - "type": "Point", - "coordinates": [-77.3079, 38.8315] - }, - "db": 65.2, - "date": "2026-04-12T14:30:00.000Z" - } ] } ``` -## Client Integration Notes +--- -For the ESP32/M5GO hardware client (`m5go/main.py`), ensure you update the `WS_URL` variable to point to the correct internal server IP. +## Database Registries -For the Web Frontend (`VoiceButton.svelte`), it uses standard Web Audio API's `ScriptProcessorNode` to bridge the Float32 arrays strictly into 16-Bit Mono over a dynamic WebSocket tunnel automatically. +* `GET /api/study-rooms/history`: Pulls the active global repository of logged architectural noise measurements captured universally within the preceding 24 hours. Data payloads correspond geographically mapping `GeoJSON` nodes to front-end Mapbox topologies. +* `GET /api/study-rooms`: Pulls generic unstructured noise lists directly unfiltered from Cosmos bounds. -```python -# In m5go/main.py -WS_URL = "ws://192.168.1.100:8000/ws/voice" -``` +> [!IMPORTANT] +> The browser frontend strictly configures standard Web Audio API's `ScriptProcessorNode` interfaces routing data synchronously to this backend! Wait to close down pipelines until *after* all WS queues have successfully been delivered. diff --git a/backend/emoji_data.py b/backend/emoji_data.py new file mode 100644 index 0000000..dfffd17 --- /dev/null +++ b/backend/emoji_data.py @@ -0,0 +1,276 @@ +"""Data containing all current emoji +Extracted from https://unicode.org/Public/emoji/latest/emoji-test.txt +and https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-variation-sequences.txt +See utils/generate_emoji.py + ++----------------+-------------+------------------+-------------------+ +| Emoji Version | Date | Unicode Version | Data File Comment | ++----------------+-------------+------------------+-------------------+ +| N/A | 2010-10-11 | Unicode 6.0 | E0.6 | +| N/A | 2014-06-16 | Unicode 7.0 | E0.7 | +| Emoji 1.0 | 2015-06-09 | Unicode 8.0 | E1.0 | +| Emoji 2.0 | 2015-11-12 | Unicode 8.0 | E2.0 | +| Emoji 3.0 | 2016-06-03 | Unicode 9.0 | E3.0 | +| Emoji 4.0 | 2016-11-22 | Unicode 9.0 | E4.0 | +| Emoji 5.0 | 2017-06-20 | Unicode 10.0 | E5.0 | +| Emoji 11.0 | 2018-05-21 | Unicode 11.0 | E11.0 | +| Emoji 12.0 | 2019-03-05 | Unicode 12.0 | E12.0 | +| Emoji 12.1 | 2019-10-21 | Unicode 12.1 | E12.1 | +| Emoji 13.0 | 2020-03-10 | Unicode 13.0 | E13.0 | +| Emoji 13.1 | 2020-09-15 | Unicode 13.0 | E13.1 | +| Emoji 14.0 | 2021-09-14 | Unicode 14.0 | E14.0 | +| Emoji 15.0 | 2022-09-13 | Unicode 15.0 | E15.0 | +| Emoji 15.1 | 2023-09-12 | Unicode 15.1 | E15.1 | +| Emoji 16.0 | 2024-09-10 | Unicode 16.0 | E16.0 | + + http://www.unicode.org/reports/tr51/ + +""" + +__all__ = ['STATUS', 'LANGUAGES'] + +from typing import Any, Dict, List + + +component = 1 +fully_qualified = 2 +minimally_qualified = 3 +unqualified = 4 + +STATUS: Dict[str, int] = { + 'component': component, + 'fully_qualified': fully_qualified, + 'minimally_qualified': minimally_qualified, + 'unqualified': unqualified, +} + +LANGUAGES: List[str] = [ + 'en', + 'es', + 'ja', + 'ko', + 'pt', + 'it', + 'fr', + 'de', + 'fa', + 'id', + 'zh', + 'ru', + 'tr', + 'ar', +] + + + + +EMOJI_DATA: Dict[str, Dict[str, Any]] = { + '\U0001f947': { + 'en': ':1st_place_medal:', + 'status': fully_qualified, + 'E': 3, + 'de': ':goldmedaille:', + 'es': ':medalla_de_oro:', + 'fr': ':médaille_d’or:', + 'ja': ':金メダル:', + 'ko': ':금메달:', + 'pt': ':medalha_de_ouro:', + 'it': ':medaglia_d’oro:', + 'fa': ':مدال_طلا:', + 'id': ':medali_emas:', + 'zh': ':金牌:', + 'ru': ':золотая_медаль:', + 'tr': ':birincilik_madalyası:', + 'ar': ':ميدالية_مركز_أول:', + }, + '\U0001f948': { + 'en': ':2nd_place_medal:', + 'status': fully_qualified, + 'E': 3, + 'de': ':silbermedaille:', + 'es': ':medalla_de_plata:', + 'fr': ':médaille_d’argent:', + 'ja': ':銀メダル:', + 'ko': ':은메달:', + 'pt': ':medalha_de_prata:', + 'it': ':medaglia_d’argento:', + 'fa': ':مدال_نقره:', + 'id': ':medali_perak:', + 'zh': ':银牌:', + 'ru': ':серебряная_медаль:', + 'tr': ':ikincilik_madalyası:', + 'ar': ':ميدالية_مركز_ثان:', + }, + '\U0001f949': { + 'en': ':3rd_place_medal:', + 'status': fully_qualified, + 'E': 3, + 'de': ':bronzemedaille:', + 'es': ':medalla_de_bronce:', + 'fr': ':médaille_de_bronze:', + 'ja': ':銅メダル:', + 'ko': ':동메달:', + 'pt': ':medalha_de_bronze:', + 'it': ':medaglia_di_bronzo:', + 'fa': ':مدال_برنز:', + 'id': ':medali_perunggu:', + 'zh': ':铜牌:', + 'ru': ':бронзовая_медаль:', + 'tr': ':üçüncülük_madalyası:', + 'ar': ':ميدالية_مركز_ثالث:', + }, + '\U0001f18e': { + 'en': ':AB_button_(blood_type):', + 'status': fully_qualified, + 'E': 0.6, + 'alias': [':ab:', ':ab_button_blood_type:'], + 'de': ':großbuchstaben_ab_in_rotem_quadrat:', + 'es': ':grupo_sanguíneo_ab:', + 'fr': ':groupe_sanguin_ab:', + 'ja': ':血液型ab型:', + 'ko': ':에이비형:', + 'pt': ':botão_ab_(tipo_sanguíneo):', + 'it': ':gruppo_sanguigno_ab:', + 'fa': ':دکمه_آ_ب_(گروه_خونی):', + 'id': ':tombol_ab_(golongan_darah):', + 'zh': ':AB型血:', + 'ru': ':IV_группа_крови:', + 'tr': ':ab_düğmesi_(kan_grubu):', + 'ar': ':زر_ab_(فئة_الدم):', + }, + '\U0001f3e7': { + 'en': ':ATM_sign:', + 'status': fully_qualified, + 'E': 0.6, + 'alias': [':atm:', ':atm_sign:'], + 'de': ':symbol_geldautomat:', + 'es': ':señal_de_cajero_automático:', + 'fr': ':distributeur_de_billets:', + 'ja': ':atm:', + 'ko': ':에이티엠:', + 'pt': ':símbolo_de_caixa_automático:', + 'it': ':simbolo_dello_sportello_bancomat:', + 'fa': ':نشان_عابربانک:', + 'id': ':tanda_atm:', + 'zh': ':取款机:', + 'ru': ':значок_банкомата:', + 'tr': ':atm_işareti:', + 'ar': ':علامة_ماكينة_صرف_آلي:', + }, + '\U0001f170\U0000fe0f': { + 'en': ':A_button_(blood_type):', + 'status': fully_qualified, + 'E': 0.6, + 'alias': [':a:', ':a_button_blood_type:'], + 'variant': True, + 'de': ':großbuchstabe_a_in_rotem_quadrat:', + 'es': ':grupo_sanguíneo_a:', + 'fr': ':groupe_sanguin_a:', + 'ja': ':血液型a型:', + 'ko': ':에이형:', + 'pt': ':botão_a_(tipo_sanguíneo):', + 'it': ':gruppo_sanguigno_a:', + 'fa': ':دکمه_آ_(گروه_خونی):', + 'id': ':tombol_a_(golongan_darah):', + 'zh': ':A型血:', + 'ru': ':ii_группа_крови:', + 'tr': ':a_düğmesi_(kan_grubu):', + 'ar': ':زر_a:', + }, + '\U0001f170': { + 'en': ':A_button_(blood_type):', + 'status': unqualified, + 'E': 0.6, + 'alias': [':a:', ':a_button_blood_type:'], + 'variant': True, + 'de': ':großbuchstabe_a_in_rotem_quadrat:', + 'es': ':grupo_sanguíneo_a:', + 'fr': ':groupe_sanguin_a:', + 'ja': ':血液型a型:', + 'ko': ':에이형:', + 'pt': ':botão_a_(tipo_sanguíneo):', + 'it': ':gruppo_sanguigno_a:', + 'fa': ':دکمه_آ_(گروه_خونی):', + 'id': ':tombol_a_(golongan_darah):', + 'zh': ':A型血:', + 'ru': ':II_группа_крови:', + 'tr': ':a_düğmesi_(kan_grubu):', + 'ar': ':زر_a:', + }, + '\U0001f1e6\U0001f1eb': { + 'en': ':Afghanistan:', + 'status': fully_qualified, + 'E': 2, + 'alias': [':flag_for_Afghanistan:', ':afghanistan:'], + 'de': ':flagge_afghanistan:', + 'es': ':bandera_afganistán:', + 'fr': ':drapeau_afghanistan:', + 'ja': ':旗_アフガニスタン:', + 'ko': ':깃발_아프가니스탄:', + 'pt': ':bandeira_afeganistão:', + 'it': ':bandiera_afghanistan:', + 'fa': ':پرچم_افغانستان:', + 'id': ':bendera_afganistan:', + 'zh': ':阿富汗:', + 'ru': ':флаг_Афганистан:', + 'tr': ':bayrak_afganistan:', + 'ar': ':علم_أفغانستان:', + }, + '\U0001f1e6\U0001f1f1': { + 'en': ':Albania:', + 'status': fully_qualified, + 'E': 2, + 'alias': [':flag_for_Albania:', ':albania:'], + 'de': ':flagge_albanien:', + 'es': ':bandera_albania:', + 'fr': ':drapeau_albanie:', + 'ja': ':旗_アルバニア:', + 'ko': ':깃발_알바니아:', + 'pt': ':bandeira_albânia:', + 'it': ':bandiera_albania:', + 'fa': ':پرچم_آلبانی:', + 'id': ':bendera_albania:', + 'zh': ':阿尔巴尼亚:', + 'ru': ':флаг_Албания:', + 'tr': ':bayrak_arnavutluk:', + 'ar': ':علم_ألبانيا:', + }, + '\U0001f1e9\U0001f1ff': { + 'en': ':Algeria:', + 'status': fully_qualified, + 'E': 2, + 'alias': [':flag_for_Algeria:', ':algeria:'], + 'de': ':flagge_algerien:', + 'es': ':bandera_argelia:', + 'fr': ':drapeau_algérie:', + 'ja': ':旗_アルジェリア:', + 'ko': ':깃발_알제리:', + 'pt': ':bandeira_argélia:', + 'it': ':bandiera_algeria:', + 'fa': ':پرچم_الجزایر:', + 'id': ':bendera_aljazair:', + 'zh': ':阿尔及利亚:', + 'ru': ':флаг_Алжир:', + 'tr': ':bayrak_cezayir:', + 'ar': ':علم_الجزائر:', + }, + '\U0001f1e6\U0001f1f8': { + 'en': ':American_Samoa:', + 'status': fully_qualified, + 'E': 2, + 'alias': [':flag_for_American_Samoa:', ':american_samoa:'], + 'de': ':flagge_amerikanisch-samoa:', + 'es': ':bandera_samoa_americana:', + 'fr': ':drapeau_samoa_américaines:', + 'ja': ':旗_米領サモア:', + 'ko': ':깃발_아메리칸_사모아:', + 'pt': ':bandeira_samoa_americana:', + 'it': ':bandiera_samoa_americane:', + 'fa': ':پرچم_ساموآی_امریکا:', + 'id': ':bendera_samoa_amerika:', + 'zh': ':美属萨摩亚:', + 'ru': ':флаг_Американское_Самоа:', + 'tr': ':bayrak_amerikan_samoası:', + 'ar': ':علم_ساموا_الأمريكية:', + }, +} diff --git a/backend/server.py b/backend/server.py index aacc420..9baf13b 100644 --- a/backend/server.py +++ b/backend/server.py @@ -8,6 +8,7 @@ import requests import json import base64 import urllib.request + # import ssl # ssl._create_default_https_context = ssl._create_unverified_context @@ -35,14 +36,14 @@ app.add_middleware( allow_headers=["*"], ) -# MongoDB Setup + import certifi MONGO_URI = os.getenv("MONGODB_URI", "mongodb://localhost:27017/") mongo_client = MongoClient(MONGO_URI, tlsCAFile=certifi.where()) db = mongo_client.study_buddy_db study_rooms_collection = db.study_rooms -# Pydantic models for Study Room Data + class GeoJSONPoint(BaseModel): type: str = "Point" coordinates: List[float] @@ -75,12 +76,12 @@ HEADERS = { "sec-fetch-site": "same-origin", "sentry-trace": "250c82a03041415b99422d838ccc7003-9a9ec11d7fd0293b-0", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0", - "x-cosmos-session-281286": "0:-1#10931", - "x-cosmos-session-295334": "0:-1#749854", - "x-cosmos-session-317755": "0:-1#191601", - "x-cosmos-session-382299": "0:-1#265024", - "x-cosmos-session-418988": "0:-1#4058856", - "x-cosmos-session-793952": "0:-1#14004", + "x-cosmos-session-281286": "0:-1", + "x-cosmos-session-295334": "0:-1", + "x-cosmos-session-317755": "0:-1", + "x-cosmos-session-382299": "0:-1", + "x-cosmos-session-418988": "0:-1", + "x-cosmos-session-793952": "0:-1", "x-request-id": "6a128b8a-7f63-4f97-a40b-bfd31b4a376e", "x-timezone": "America/New_York", } @@ -97,7 +98,7 @@ def _write_wav_to_buffer(pcm_data: bytes, sample_rate: int = SAMPLE_RATE) -> byt buf.write(b"WAVE") buf.write(b"fmt ") buf.write(struct.pack(" str: """Transcribe raw PCM audio using faster-whisper via a temp WAV file.""" from faster_whisper import WhisperModel print(f" Using sample rate: {sample_rate} Hz") - - # Debug: analyze PCM audio quality + + num_samples = len(pcm_data) // 2 if num_samples > 0: samples = list(struct.unpack(f"<{num_samples}h", pcm_data[:num_samples * 2])) @@ -121,30 +122,30 @@ def _transcribe_pcm(pcm_data: bytes, sample_rate: int = SAMPLE_RATE) -> str: mean_s = sum(samples) / num_samples rms = (sum(s * s for s in samples) / num_samples) ** 0.5 print(f" PCM stats (raw): {num_samples} samples, min={min_s}, max={max_s}, mean={mean_s:.1f}, RMS={rms:.1f}") - - # Remove DC offset (center audio at 0) + + dc_offset = int(round(mean_s)) samples = [max(-32768, min(32767, s - dc_offset)) for s in samples] pcm_data = struct.pack(f"<{num_samples}h", *samples) - - # Stats after correction + + rms_fixed = (sum(s * s for s in samples) / num_samples) ** 0.5 print(f" PCM stats (fixed): DC offset removed={dc_offset}, RMS={rms_fixed:.1f}") - + wav_data = _write_wav_to_buffer(pcm_data, sample_rate=sample_rate) tmp_fd, tmp_path = tempfile.mkstemp(suffix=".wav") try: with os.fdopen(tmp_fd, "wb") as f: f.write(wav_data) - - # Save a debug copy so we can listen + + debug_path = os.path.join(os.path.dirname(__file__), "debug_audio.wav") with open(debug_path, "wb") as df: df.write(wav_data) print(f" Debug WAV saved to: {debug_path}") - # Initialize the model (using base model for speed) + model = WhisperModel("base", device="cpu", compute_type="int8") segments, info = model.transcribe(tmp_path, beam_size=5) seg_list = list(segments) @@ -190,7 +191,7 @@ def get_terp_ai_response(message: str) -> str: except Exception as e: print(f"Terp AI error: {e}") return "I am sorry, there was an error connecting to Terp AI." - + return full_response def _convert_to_pcm(audio_data: bytes, input_format: str = "mp3") -> bytes | None: @@ -232,7 +233,7 @@ def _generate_tts(text: str) -> bytes | None: print("ElevenLabs API key not configured") return None - # Request PCM directly — no ffmpeg needed + url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}?output_format=pcm_16000" headers = { @@ -267,7 +268,7 @@ def _generate_tts(text: str) -> bytes | None: print(f"ElevenLabs TTS error: {e}") return None -# Latest TTS WAV stored in memory for HTTP download by M5GO + _latest_tts_wav = None @app.get("/api/tts-audio") @@ -282,40 +283,40 @@ async def websocket_voice(websocket: WebSocket): await websocket.accept() print("Device connected to WebSocket.") audio_buffer = bytearray() - + try: while True: data = await websocket.receive() - + if "bytes" in data: audio_buffer.extend(data["bytes"]) - + elif "text" in data: try: msg = json.loads(data["text"]) if msg.get("event") == "stop_listening": pcm_data = bytes(audio_buffer) - audio_buffer = bytearray() # Reset for next time + audio_buffer = bytearray() device_sample_rate = msg.get("sample_rate", SAMPLE_RATE) - + print(f"Received stop_listening event. Buffer size: {len(pcm_data)} bytes, sample_rate: {device_sample_rate} Hz") - + if len(pcm_data) < 3200: print("Audio too short, ignoring.") await websocket.send_bytes(b"") continue - - # Step 1: Speech to Text + + print("Transcribing...") user_text = _transcribe_pcm(pcm_data, sample_rate=device_sample_rate) if not user_text: print("Transcription failed or empty.") await websocket.send_text(json.dumps({"event": "error", "msg": "No speech detected"})) continue - + print(f"User said: {user_text}") - - # Step 2: Terp AI + + print("Sending to Terp AI...") context_str = get_latest_locations_context() augmented_prompt = f"USER ASKS: {user_text}\n\n[SYSTEM CONTEXT - LATEST UMD ROOM STATS TO HELP YOU ANSWER IF ASKED]:\n{context_str}" @@ -324,15 +325,15 @@ async def websocket_voice(websocket: WebSocket): print("No response from Terp AI.") await websocket.send_text(json.dumps({"event": "error", "msg": "No AI response"})) continue - + print(f"Terp AI response: {ai_response_text}") - - # Step 3: Text to Speech + + print("Generating TTS...") tts_pcm = _generate_tts(ai_response_text) - + if tts_pcm: - # Save as WAV for HTTP download by M5GO + global _latest_tts_wav _latest_tts_wav = _write_wav_to_buffer(tts_pcm) print(f"TTS WAV ready: {len(_latest_tts_wav)} bytes, serving via /api/tts-audio") @@ -343,7 +344,7 @@ async def websocket_voice(websocket: WebSocket): else: print("TTS failed.") await websocket.send_text(json.dumps({"event": "error", "msg": "TTS failed"})) - + except json.JSONDecodeError: pass except Exception as e: @@ -383,41 +384,41 @@ def get_latest_locations_context() -> str: }} ] latest_stats = list(study_rooms_collection.aggregate(pipeline)) - + if not latest_stats: return "No recent location noise stats available today." - + room_dict = {loc["id"]: loc["name"] for loc in UMD_LOCATIONS} - + lines = ["Latest Study Room Stats:"] for stat in latest_stats: room_id = stat.get("_id") name = room_dict.get(room_id, room_id) db = stat.get("latest_db", 0.0) - + status = "Quiet" if isinstance(db, (int, float)): if db >= 65: status = "Loud" elif db >= 55: status = "Moderate" - + lines.append(f"- {name}: Noise Level {db:.1f} dB ({status})") - + return "\n".join(lines) @app.post("/api/study-rooms") async def create_study_room_data(data: StudyRoomData): - # Check if the coordinates match one of the known locations (with small tolerance) + is_valid_location = False req_lng, req_lat = data.location.coordinates[0], data.location.coordinates[1] - + for loc in UMD_LOCATIONS: if abs(loc["lng"] - req_lng) < 0.0001 and abs(loc["lat"] - req_lat) < 0.0001: is_valid_location = True - # Override coordinates to exactly match known location for consistency + data.location.coordinates = [loc["lng"], loc["lat"]] data.room_id = loc["id"] break - + if not is_valid_location: from fastapi import HTTPException raise HTTPException(status_code=400, detail="Invalid location. Coordinates must correspond to a known UMD location.") @@ -438,7 +439,7 @@ async def get_study_room_history(): """Get all study room data from the last 24 hours.""" twenty_four_hours_ago = datetime.utcnow() - timedelta(hours=24) rooms = list(study_rooms_collection.find( - {"date": {"$gte": twenty_four_hours_ago}}, + {"date": {"$gte": twenty_four_hours_ago}}, {"_id": 0} ).sort("date", -1)) return {"data": rooms} @@ -448,13 +449,13 @@ async def serve_spa(full_path: str): static_dir = "static" if not os.path.exists(static_dir): return {"error": "Static directory not found. Please build the frontend."} - + static_path = os.path.join(static_dir, full_path) if os.path.isfile(static_path): return FileResponse(static_path) - + index_path = os.path.join(static_dir, "index.html") if os.path.exists(index_path): return FileResponse(index_path) - + return {"error": "index.html not found in static directory"} diff --git a/backend/vision.py b/backend/vision.py index 9f4d499..6eabd0d 100644 --- a/backend/vision.py +++ b/backend/vision.py @@ -4,15 +4,15 @@ from ultralytics import YOLO from scipy.spatial.distance import cdist from scipy.optimize import linear_sum_assignment -# Load YOLOv8 nano model (downloads automatically if not found) -# 'yolov8n.pt' is lightweight and fast for this purpose + + try: model = YOLO("yolov8n.pt") except Exception as e: print(f"Error loading YOLO model: {e}") model = None -# COCO Class IDs + PERSON_CLASS_ID = 0 CHAIR_CLASS_ID = 56 @@ -20,16 +20,16 @@ def analyze_room_image(image_bytes: bytes): if not model: return {"error": "Vision model is not loaded"} - # Convert bytes to numpy array then to cv2 image + nparr = np.frombuffer(image_bytes, np.uint8) img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) - + if img is None: return {"error": "Invalid image format"} - # Run inference + results = model(img) - + people = [] chairs = [] @@ -38,25 +38,25 @@ def analyze_room_image(image_bytes: bytes): for box in boxes: cls_id = int(box.cls[0]) conf = float(box.conf[0]) - - # Extract center of bounding box + + x1, y1, x2, y2 = box.xyxy[0] cx = (x1 + x2) / 2.0 cy = (y1 + y2) / 2.0 centroid = [float(cx), float(cy)] - # Only consider detections with confidence > 0.3 + if conf > 0.3: if cls_id == PERSON_CLASS_ID: people.append({ - "centroid": centroid, - "box": [float(x1), float(y1), float(x2), float(y2)], + "centroid": centroid, + "box": [float(x1), float(y1), float(x2), float(y2)], "conf": conf }) elif cls_id == CHAIR_CLASS_ID: chairs.append({ - "centroid": centroid, - "box": [float(x1), float(y1), float(x2), float(y2)], + "centroid": centroid, + "box": [float(x1), float(y1), float(x2), float(y2)], "conf": conf }) @@ -64,18 +64,18 @@ def analyze_room_image(image_bytes: bytes): num_chairs = len(chairs) pairs = [] - - # Bipartite matching if both people and chairs exist + + if num_people > 0 and num_chairs > 0: people_coords = [p["centroid"] for p in people] chairs_coords = [c["centroid"] for c in chairs] - - # Distance matrix (Euclidean distances) + + dist_matrix = cdist(people_coords, chairs_coords, metric='euclidean') - - # Hungarian algorithm to minimize total distance for pairings + + row_ind, col_ind = linear_sum_assignment(dist_matrix) - + for person_idx, chair_idx in zip(row_ind, col_ind): distance = float(dist_matrix[person_idx, chair_idx]) pairs.append({ @@ -84,8 +84,8 @@ def analyze_room_image(image_bytes: bytes): "distance": distance }) - # Basic logic: room is full if there are at least as many people as chairs. - # Can be adjusted based on specific room definitions + + is_full = num_people >= num_chairs if num_chairs > 0 else False return { diff --git a/m5go/main.py b/m5go/main.py index eed4345..4af2df0 100644 --- a/m5go/main.py +++ b/m5go/main.py @@ -34,26 +34,26 @@ def d_s(lcd, f, s_x, s_y, p_s): setScreenColor(0x222222) -# ========================================== -# CONFIGURATION -# ========================================== + + + WIFI_SSID = "Blobby" WIFI_PASS = "73556088" WS_URL = "ws://192.168.137.1:8000/ws/voice" -# Location Settings for Noise Monitoring + CURRENT_ROOM_ID = "mckeldin" CURRENT_LAT = 38.986021 CURRENT_LNG = -76.944949 -# Audio Settings -TARGET_SAMPLE_RATE = 8000 # Voice recording sample rate + +TARGET_SAMPLE_RATE = 8000 AUDIO_CHUNK_SIZE = 2048 -# ========================================== -# WEBSOCKET CLIENT -# ========================================== + + + class WSClient: def __init__(self, sock): @@ -167,12 +167,12 @@ def ws_connect(url): else: host = host_port port = 80 - + addr = usocket.getaddrinfo(host, port)[0][-1] sock = usocket.socket() sock.connect(addr) sock.settimeout(15) - + key = ubinascii.b2a_base64(os.urandom(16)).strip().decode() req = ( "GET %s HTTP/1.1\r\n" @@ -183,7 +183,7 @@ def ws_connect(url): "Sec-WebSocket-Version: 13\r\n" "\r\n" ) % (path, host_port, key) - + sock.send(req.encode()) resp = b"" while b"\r\n\r\n" not in resp: @@ -192,17 +192,17 @@ def ws_connect(url): sock.close() raise Exception("Closed during handshake") resp += b - + status_line = resp.split(b"\r\n")[0] if b"101" not in status_line: sock.close() raise Exception("Upgrade failed: " + status_line.decode()) - + return WSClient(sock) -# ========================================== -# UI & UTILITY FUNCTIONS -# ========================================== + + + _cur_face = None def set_face(face): @@ -224,12 +224,12 @@ def connect_wifi(): wlan.active(True) if not wlan.isconnected(): wlan.connect(WIFI_SSID, WIFI_PASS) - + attempts = 0 while not wlan.isconnected() and attempts < 20: time.sleep(0.5) attempts += 1 - + if wlan.isconnected(): lcd.clear() lcd.print("WiFi Connected!", 0, 0, 0x00FF00) @@ -243,9 +243,9 @@ def connect_wifi(): lcd.clear() return wlan.isconnected() -# ========================================== -# AUDIO I/O -# ========================================== + + + def get_adc(): try: @@ -265,7 +265,7 @@ def get_db(adc_obj): sum_v = 0 sum_sq = 0 count = 0 - end_t = time.ticks_ms() + 40 + end_t = time.ticks_ms() + 40 while time.ticks_ms() < end_t: try: v = adc_obj.read() @@ -275,29 +275,29 @@ def get_db(adc_obj): except: pass if count == 0: return 30 - + mean = sum_v / count variance = (sum_sq / count) - (mean * mean) - + if variance <= 1: return 30 amp = math.sqrt(variance) if amp <= 1: return 30 - - # Scale to human DB + + return 20 * math.log10(amp) + 25 def init_manual_spk(): try: - # M5GO Core speaker sits on DAC 1 (Pin 25) - # Therefore we MUST strictly use DAC_BUILT_IN. Digital I2S on this pin is not supported! + + if hasattr(machine.I2S, "MODE_DAC_BUILT_IN"): mode = getattr(machine.I2S, "MODE_MASTER", 1) | getattr(machine.I2S, "MODE_TX", 2) | getattr(machine.I2S, "MODE_DAC_BUILT_IN", 0) cfmt = getattr(machine.I2S, "CHANNEL_FMT_RIGHT_LEFT", 1) dfmt = getattr(machine.I2S, "FORMAT_I2S_MSB", 1) - + i2s_id = getattr(machine.I2S, "NUM0", 0) - - # Different MicroPython versions vary wildly on kwarg vs positional I2S init structure + + try_sigs = [ ([i2s_id], {"mode": mode, "rate": 16000, "bits": 16, "format": cfmt, "ibuf": 2048}), ([i2s_id, mode, 16000, 16, cfmt, dfmt], {}), @@ -307,14 +307,14 @@ def init_manual_spk(): ([], {"mode": mode, "sample_rate": 16000, "bits": 16}), ([i2s_id], {"mode": mode, "rate": 16000, "bits": 16}) ] - + last_err = None for args, kwargs in try_sigs: try: return machine.I2S(*args, **kwargs) except Exception as e: last_err = e - + print("I2S Fallback err (exhausted):", last_err) return None else: @@ -330,34 +330,34 @@ def stream_http_audio(url): if not audio_out: print("Speaker init failed, cannot play audio via DAC") return False - + if url.startswith("http://"): rest = url[7:] else: raise ValueError("Only http:// supported") - + if "/" in rest: host_port = rest.split("/", 1)[0] path = "/" + rest.split("/", 1)[1] else: host_port = rest path = "/" - + if ":" in host_port: host = host_port.split(":")[0] port = int(host_port.split(":")[1]) else: host = host_port port = 80 - + addr = usocket.getaddrinfo(host, port)[0][-1] sock = usocket.socket() sock.connect(addr) sock.settimeout(30) - + req = "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (path, host_port) sock.send(req.encode()) - + hdr = b"" while b"\r\n\r\n" not in hdr: b = sock.recv(1) @@ -365,17 +365,17 @@ def stream_http_audio(url): sock.close() return False hdr += b - - # Read WAV header + + header_left = 44 while header_left > 0: chunk = sock.recv(header_left) if not chunk: break header_left -= len(chunk) - + in_buf = bytearray(1024) out_buf = bytearray(2048) - + while True: n = 0 while n < 1024: @@ -383,52 +383,52 @@ def stream_http_audio(url): if not chunk: break in_buf[n:n+len(chunk)] = chunk n += len(chunk) - + if n == 0: break - + samples = n // 2 for j in range(samples): idx = j * 2 - # Read 16-bit Signed LE + s = in_buf[idx] | (in_buf[idx + 1] << 8) if s >= 32768: s -= 65536 - - # Convert to Unsigned + Center Offset for DAC + + u = (s + 32768) & 0xFFFF u_lo = u & 0xFF u_hi = u >> 8 - - # Map Stereo for built-in MSB + + o_idx = j * 4 out_buf[o_idx] = u_lo out_buf[o_idx + 1] = u_hi out_buf[o_idx + 2] = u_lo out_buf[o_idx + 3] = u_hi - + try: audio_out.write(out_buf[:samples * 4]) except Exception as e: print("Write err:", e) break - + sock.close() if hasattr(audio_out, 'deinit'): audio_out.deinit() return True -# ========================================== -# MAIN LOOP -# ========================================== + + + def run_main(): if not connect_wifi(): return - + adc = get_adc() ws = None l_db_s = "" s_db = 30.0 last_db_post_time = time.ticks_ms() - + draw_status("Hold Button A to Talk", 0xFFFFFF, f_s_o) def maintain_ws(): @@ -445,31 +445,31 @@ def run_main(): while True: gc.collect() - # --- Voice Interaction --- + if btnA.isPressed(): if maintain_ws(): draw_status("Listening...", 0x0000FF, f_listen) - + send_buf = bytearray(AUDIO_CHUNK_SIZE) buf_pos = 0 total_samples = 0 - + rec_start_us = time.ticks_us() - - # Fastest possible analog capture loop + + while btnA.isPressed(): try: raw = adc.read() if adc else 2048 sample = (raw - 2048) * 16 - # Fast clamp + if sample > 32767: sample = 32767 elif sample < -32768: sample = -32768 - + send_buf[buf_pos] = sample & 0xFF send_buf[buf_pos + 1] = (sample >> 8) & 0xFF buf_pos += 2 total_samples += 1 - + if buf_pos >= AUDIO_CHUNK_SIZE: if ws: ws.send(bytes(send_buf)) buf_pos = 0 @@ -482,31 +482,31 @@ def run_main(): ws.send(bytes(send_buf[:buf_pos])) except: pass - + draw_status("Thinking...", 0xFFFF00, f_t_1) actual_rate = (total_samples * 1000000) // time.ticks_diff(time.ticks_us(), rec_start_us) print("Captured at", actual_rate, "Hz") - + try: ws.send(json.dumps({"event": "stop_listening", "sample_rate": actual_rate})) - - # Wait for TTS ready + + resp = ws.recv() if resp and isinstance(resp, str): msg = json.loads(resp) if msg.get("event") == "tts_ready": - # Start direct TCP stream immediately + draw_status("Speaking...", 0x00FF00, f_speak) http_base = WS_URL.replace("ws://", "http://").replace("/ws/voice", "") audio_url = http_base + "/api/tts-audio" - + try: stream_http_audio(audio_url) except Exception as e: print("Stream Err:", e) draw_status("Play Failed", 0xFF0000, f_s_c) time.sleep(1) - + elif msg.get("event") == "error": draw_status("Error: " + msg.get("msg", "")[:10], 0xFF0000, f_s_c) time.sleep(2) @@ -515,14 +515,14 @@ def run_main(): try: ws.close() except: pass ws = None - + draw_status("Hold Button A to Talk", 0xFFFFFF, f_s_o) - # --- Noise Monitoring --- + r_db = get_db(adc) s_db = (s_db * 0.8) + (r_db * 0.2) db_val = int(s_db) - + i_c = 0x89b4fa if db_val < 40 else (0x94e2d5 if db_val < 55 else (0xf9e2af if db_val < 65 else (0xfab387 if db_val < 80 else 0xf38ba8))) lcd.fillRect(0, 0, 320, 4, i_c) db_s = "Noise: %d dB" % db_val @@ -530,16 +530,16 @@ def run_main(): lcd.fillRect(0, 4, 150, 15, 0x222222) lcd.print(db_s, 5, 4, i_c) l_db_s = db_s - - # Periodic DB Posting + + try: now_ms = time.ticks_ms() if time.ticks_diff(now_ms, last_db_post_time) > 15000: last_db_post_time = now_ms payload_str = '{"room_id":"%s","location":{"type":"Point","coordinates":[%s,%s]},"db":%s}' % (CURRENT_ROOM_ID, CURRENT_LNG, CURRENT_LAT, db_val) http_url = WS_URL.replace("ws://", "http://").replace("/ws/voice", "/api/study-rooms") - - # Raw socket post + + h_p = http_url.split("://")[1].split("/")[0] p_th = "/" + http_url.split("://")[1].split("/", 1)[1] if "/" in http_url.split("://")[1] else "/" h_b = h_p.split(":")[0] @@ -554,8 +554,8 @@ def run_main(): del s, req, payload_str except: pass - - # Idle Face Blinking logic + + if db_val > 67: set_face(f_angry) else: @@ -564,7 +564,7 @@ def run_main(): set_face(f_s_c) else: set_face(f_s_o) - + time.sleep(0.02) if __name__ == "__main__": diff --git a/scripts/ai.py b/scripts/ai.py index 8a190d4..e2d9cf9 100755 --- a/scripts/ai.py +++ b/scripts/ai.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 + import base64 import urllib.request import urllib.error @@ -28,16 +28,16 @@ HEADERS = { "sec-fetch-mode": "cors", "sec-fetch-site": "same-origin", "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", - "x-cosmos-session-281286": "0:-1#10931", - "x-cosmos-session-295334": "0:-1#745295", - "x-cosmos-session-317755": "0:-1#190559", - "x-cosmos-session-382299": "0:-1#264026", - "x-cosmos-session-418988": "0:-1#4052014", - "x-cosmos-session-793952": "0:-1#13832", + "x-cosmos-session-281286": "0:-1 + "x-cosmos-session-295334": "0:-1 + "x-cosmos-session-317755": "0:-1 + "x-cosmos-session-382299": "0:-1 + "x-cosmos-session-418988": "0:-1 + "x-cosmos-session-793952": "0:-1 "x-timezone": "America/New_York", } -# The parent segment ID to chain from — update this to your latest segment + INITIAL_PARENT_SEGMENT_ID = "d7765a66-a24b-4620-b003-11b4c1ed2c36" @@ -59,7 +59,7 @@ def send_message(message: str, conversation_id: str = CONVERSATION_ID, parent_se payload = json.dumps(body).encode("utf-8") - # Generate fresh per-request headers + sentry_trace_id = uuid.uuid4().hex sentry_span_id = uuid.uuid4().hex[:16] headers = dict(HEADERS) @@ -105,7 +105,7 @@ def send_message(message: str, conversation_id: str = CONVERSATION_ID, parent_se print(f"\n[IDs: {decoded}]", file=sys.stderr) try: ids = json.loads(decoded) - # Try common key names for the segment ID + new_segment_id = ( ids.get("segmentId") or ids.get("id") diff --git a/scripts/generate_fake_data.py b/scripts/generate_fake_data.py index e1c0b69..42f435e 100644 --- a/scripts/generate_fake_data.py +++ b/scripts/generate_fake_data.py @@ -2,7 +2,7 @@ import random from datetime import datetime, timedelta from pymongo import MongoClient -# MongoDB Setup + MONGO_URI = "mongodb+srv://SarayuJ:SarayuJ123@cluster0.xjy5c.mongodb.net/testing" client = MongoClient(MONGO_URI) db = client.study_buddy_db @@ -25,65 +25,65 @@ def get_db_for_time_and_location(hour, loc_id): Generate a dB level based on the hour of the day and the location. This creates a recognizable pattern for AI analysis. """ - base_db = 40.0 # Ambient noise - + base_db = 40.0 + if loc_id in ['mckeldin', 'esj']: - # Busy during the day (10am - 4pm) + if 10 <= hour <= 16: base_db = 75.0 elif 17 <= hour <= 22: base_db = 60.0 else: base_db = 45.0 - + elif loc_id in ['stem', 'iribe']: - # Busy in the afternoon/evening (2pm - 8pm) + if 14 <= hour <= 20: base_db = 70.0 elif 9 <= hour <= 13: base_db = 55.0 else: base_db = 42.0 - + elif loc_id == 'stamp': - # Busy during lunch (12pm - 2pm) and dinner (5pm - 7pm) + if 12 <= hour <= 14 or 17 <= hour <= 19: base_db = 85.0 elif 10 <= hour <= 21: base_db = 65.0 else: base_db = 50.0 - + else: - # General locations (Clarice, Yahentamitsi, Reckord, Hornbake) - # Moderate noise during the day + + if 9 <= hour <= 18: base_db = 60.0 else: base_db = 45.0 - - # Add random noise to make it look realistic (+/- 5 dB) + + noise = random.uniform(-5.0, 5.0) return max(30.0, min(100.0, base_db + noise)) def generate_fake_data(): print("Clearing existing study room data...") collection.delete_many({}) - + now = datetime.utcnow() start_time = now - timedelta(hours=24) - + docs_to_insert = [] - + print("Generating 24 hours of fake data with patterns...") - # Generate data points every 15 minutes for the last 24 hours + current_time = start_time while current_time <= now: hour = current_time.hour - + for loc in UMD_LOCATIONS: db_level = get_db_for_time_and_location(hour, loc["id"]) - + doc = { "room_id": loc["id"], "location": { @@ -94,9 +94,9 @@ def generate_fake_data(): "date": current_time } docs_to_insert.append(doc) - + current_time += timedelta(minutes=15) - + print(f"Inserting {len(docs_to_insert)} records into MongoDB...") collection.insert_many(docs_to_insert) print("Done!") diff --git a/website/src/app.d.ts b/website/src/app.d.ts index da08e6d..78e91c3 100644 --- a/website/src/app.d.ts +++ b/website/src/app.d.ts @@ -1,12 +1,12 @@ // See https://svelte.dev/docs/kit/types#app.d.ts -// for information about these interfaces + declare global { namespace App { - // interface Error {} - // interface Locals {} - // interface PageData {} - // interface PageState {} - // interface Platform {} + + + + + } } diff --git a/website/src/lib/components/AICallModal.svelte b/website/src/lib/components/AICallModal.svelte index 77abfcb..d910cd3 100644 --- a/website/src/lib/components/AICallModal.svelte +++ b/website/src/lib/components/AICallModal.svelte @@ -5,7 +5,7 @@ type CallState = 'idle' | 'listening' | 'processing' | 'speaking'; let callState = $state('idle'); - + let ws: WebSocket | null = null; let stream: MediaStream | null = null; let audioContext: AudioContext | null = null; @@ -16,10 +16,10 @@ let silenceTime = 0; let errorMessage = $state(''); let hardwareSampleRate = 16000; - - // Visualizer data + + let currentRms = $state(0); - + function cleanupMic() { try { if (processor) { @@ -44,15 +44,15 @@ function playTTS(url: string) { callState = 'speaking'; currentAudio = new Audio(url + "?t=" + Date.now()); - currentRms = 0.06; // Set a much smaller safe static visualizer size + currentRms = 0.06; currentAudio.onended = () => { currentRms = 0; if (callState === 'speaking') { - // AI is done talking, start listening automatically! + startListeningPhase(); } }; - // Some browsers require explicit play tracking + const playPromise = currentAudio.play(); if (playPromise !== undefined) { playPromise.catch(e => { @@ -72,7 +72,7 @@ currentRms = 0; try { - // Synchronous audio context resume + if (audioContext && audioContext.state === 'suspended') { await audioContext.resume(); } @@ -80,27 +80,27 @@ if (!stream) { stream = await navigator.mediaDevices.getUserMedia({ audio: true }); if (!audioContext) return; - + const source = audioContext.createMediaStreamSource(stream); processor = audioContext.createScriptProcessor(2048, 1, 1); - + processor.onaudioprocess = (e) => { if (!ws || ws.readyState !== WebSocket.OPEN || callState !== 'listening') return; - + const float32 = e.inputBuffer.getChannelData(0); const int16 = new Int16Array(float32.length); let sumSq = 0; - + for (let i = 0; i < float32.length; i++) { const s = Math.max(-1, Math.min(1, float32[i])); int16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; sumSq += s * s; } - + ws.send(int16.buffer); const rms = Math.sqrt(sumSq / float32.length); - currentRms = rms; // Drive the visualizer UI + currentRms = rms; if (rms > 0.035) { hasSpoken = true; @@ -132,15 +132,15 @@ if (ws && ws.readyState === WebSocket.OPEN) { ws.send(JSON.stringify({ event: 'stop_listening', sample_rate: hardwareSampleRate })); } - // Do NOT cleanup hardware mic here as the session is perfectly continuous! + } function startCall() { if (callState !== 'idle') return; - callState = 'listening'; // transition state instantly - + callState = 'listening'; + try { - // MUST CREATE AUDIO CONTEXT SYNCHRONOUSLY IN CLICK HANDLER + const AC = window.AudioContext || (window as any).webkitAudioContext; audioContext = new AC(); hardwareSampleRate = audioContext.sampleRate; @@ -152,7 +152,6 @@ const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; const host = window.location.hostname; ws = new WebSocket(`${protocol}//${host}:8000/ws/voice`); - ws.onmessage = (e) => { const data = JSON.parse(e.data); if (data.event === 'tts_ready') { @@ -189,7 +188,7 @@ if (callState === 'idle') { startCall(); } else if (callState === 'listening') { - // Force manual send + hasSpoken = true; finishUtterance(); } else { @@ -204,7 +203,7 @@ } onMount(() => { - // Auto-start the call when modal opens + startCall(); }); @@ -215,7 +214,7 @@
- +

Live AI Assistant

@@ -234,20 +233,20 @@

{#if callState === 'listening' || callState === 'speaking'} -
-
{/if} - +
{#if callState === 'processing'} @@ -271,21 +270,21 @@
{#if callState === 'idle'} - {:else if callState === 'listening'} - {/if} - -
- +
{#each alertsState.activeAlerts as alert (alert.id)}
@@ -33,8 +33,8 @@

{alert.loc}

{alert.time}

-
{/each} - + {#if alertsState.activeAlerts.length === 0}
@@ -63,8 +63,8 @@

NOISE SPIKE: {alert.level} dB

- - +
- + - { if (isPlaying) togglePlay(); }} class="w-full absolute opacity-0 cursor-pointer h-full z-20" /> - +
- +

24H Window

diff --git a/website/src/routes/layout.css b/website/src/routes/layout.css index 88b2893..b1325de 100644 --- a/website/src/routes/layout.css +++ b/website/src/routes/layout.css @@ -4,7 +4,7 @@ --font-sans: 'Comic Relief', 'Comic Neue', 'Comic Sans MS', cursive, system-ui, sans-serif; --font-display: 'Comic Relief', 'Comic Neue', 'Comic Sans MS', cursive, system-ui, sans-serif; - /* Catppuccin Theme */ + --color-crust: #11111b; --color-mantle: #181825; --color-base: #1e1e2e; diff --git a/website/src/routes/settings/+page.svelte b/website/src/routes/settings/+page.svelte index dc283cd..8e88b8b 100644 --- a/website/src/routes/settings/+page.svelte +++ b/website/src/routes/settings/+page.svelte @@ -46,7 +46,7 @@
-
+
@@ -62,7 +62,7 @@ Appearance and Accesibility - +

@@ -70,7 +70,7 @@ Light / Dark Mode

- +
- +
- +