Merge branch 'main' of https://github.com/GamerBoss101/HooHacks-12

Backend/index.html

@@ -274,7 +274,6 @@
             const whisperStatus = document.getElementById('whisper-status');
             const csmStatus = document.getElementById('csm-status');
             const llmStatus = document.getElementById('llm-status');
-            const webrtcStatus = document.getElementById('webrtc-status');
             const micAnimation = document.getElementById('mic-animation');
             const loadingDiv = document.getElementById('loading');
             const loadingText = document.getElementById('loading-text');
@@ -286,14 +285,7 @@
             let isAiSpeaking = false;
             let audioContext;
             let mediaStream;
-            let audioRecorder;
             let audioProcessor;
-            const audioChunks = [];
-            
-            // WebRTC variables
-            let peerConnection;
-            let dataChannel;
-            let hasActiveConnection = false;
             
             // Audio playback
             let audioQueue = [];
@@ -302,7 +294,6 @@
             // Configuration variables
             let serverSampleRate = 24000;
             let clientSampleRate = 44100;
-            let iceServers = [];
             
             // Initialize the application
             initApp();
@@ -329,7 +320,6 @@
                     updateConnectionStatus('disconnected');
                     isConnected = false;
                     cleanupAudio();
-                    cleanupWebRTC();
                 });
                 
                 socket.on('session_ready', (data) => {
@@ -337,11 +327,13 @@
                     updateModelStatus(data);
                     clientSampleRate = data.client_sample_rate;
                     serverSampleRate = data.server_sample_rate;
-                    iceServers = data.ice_servers;
                     
-                    // Initialize WebRTC if models are available
+                    // Enable start button if models are available
-                    if (data.whisper_available && data.llm_available) {
+                    if (data.whisper_available && data.csm_available) {
-                        initializeWebRTC();
+                        startButton.disabled = false;
+                        addInfoMessage('Ready for conversation. Click "Start Listening" to begin.');
+                    } else {
+                        addInfoMessage('Some models are not available. Voice chat might not work properly.');
                     }
                 });
                 
@@ -351,10 +343,6 @@
                     addInfoMessage('Ready for conversation. Click "Start Listening" to begin.');
                 });
                 
-                socket.on('webrtc_signal', (data) => {
-                    handleWebRTCSignal(data);
-                });
-                
                 socket.on('transcription', (data) => {
                     console.log('Transcription:', data);
                     addUserMessage(data.text);
@@ -460,98 +448,6 @@
                 llmStatus.style.color = data.llm_available ? 'green' : 'red';
             }
             
-            // Initialize WebRTC connection
-            function initializeWebRTC() {
-                if (!isConnected) return;
-                
-                const configuration = {
-                    iceServers: iceServers
-                };
-                
-                peerConnection = new RTCPeerConnection(configuration);
-                
-                // Create data channel for WebRTC communication
-                dataChannel = peerConnection.createDataChannel('audioData', {
-                    ordered: true
-                });
-                
-                dataChannel.onopen = () => {
-                    console.log('WebRTC data channel open');
-                    hasActiveConnection = true;
-                    webrtcStatus.textContent = 'Connected';
-                    webrtcStatus.style.color = 'green';
-                    socket.emit('webrtc_connected', { status: 'connected' });
-                };
-                
-                dataChannel.onclose = () => {
-                    console.log('WebRTC data channel closed');
-                    hasActiveConnection = false;
-                    webrtcStatus.textContent = 'Disconnected';
-                    webrtcStatus.style.color = 'red';
-                };
-                
-                // Handle ICE candidates
-                peerConnection.onicecandidate = (event) => {
-                    if (event.candidate) {
-                        socket.emit('webrtc_signal', {
-                            type: 'ice_candidate',
-                            candidate: event.candidate
-                        });
-                    }
-                };
-                
-                // Log ICE connection state changes
-                peerConnection.oniceconnectionstatechange = () => {
-                    console.log('ICE connection state:', peerConnection.iceConnectionState);
-                };
-                
-                // Create offer
-                peerConnection.createOffer()
-                    .then(offer => peerConnection.setLocalDescription(offer))
-                    .then(() => {
-                        socket.emit('webrtc_signal', {
-                            type: 'offer',
-                            sdp: peerConnection.localDescription
-                        });
-                    })
-                    .catch(error => {
-                        console.error('Error creating WebRTC offer:', error);
-                        webrtcStatus.textContent = 'Failed to Connect';
-                        webrtcStatus.style.color = 'red';
-                    });
-            }
-            
-            // Handle WebRTC signals from the server
-            function handleWebRTCSignal(data) {
-                if (!peerConnection) return;
-                
-                if (data.type === 'answer') {
-                    peerConnection.setRemoteDescription(new RTCSessionDescription(data.sdp))
-                        .catch(error => console.error('Error setting remote description:', error));
-                } 
-                else if (data.type === 'ice_candidate') {
-                    peerConnection.addIceCandidate(new RTCIceCandidate(data.candidate))
-                        .catch(error => console.error('Error adding ICE candidate:', error));
-                }
-            }
-            
-            // Clean up WebRTC connection
-            function cleanupWebRTC() {
-                if (dataChannel) {
-                    dataChannel.close();
-                }
-                
-                if (peerConnection) {
-                    peerConnection.close();
-                }
-                
-                dataChannel = null;
-                peerConnection = null;
-                hasActiveConnection = false;
-                webrtcStatus.textContent = 'Not Connected';
-                webrtcStatus.style.color = 'red';
-            }
-            
             // Toggle audio listening
             function toggleListening() {
                 if (isListening) {
@@ -648,8 +544,6 @@
                 if (audioContext && audioContext.state !== 'closed') {
                     audioContext.close().catch(error => console.error('Error closing AudioContext:', error));
                 }
-                
-                audioChunks.length = 0;
             }
             
             // Convert Float32Array to Int16Array for sending to server
@@ -669,7 +563,7 @@
                 // Convert to base64 for transmission
                 const base64Audio = arrayBufferToBase64(audioData.buffer);
                 
-                // Send via Socket.IO (could use WebRTC's DataChannel for lower latency in production)
+                // Send via Socket.IO
                 socket.emit('audio_stream', { audio: base64Audio });
             }
             

Backend/server.py

@@ -152,7 +152,10 @@ def index():
     """Serve the main interface"""
     return render_template('index.html')
 
-
+@app.route('/static/js/voice-chat.js')
+def serve_voice_chat_js():
+    """Serve the JavaScript file"""
+    return app.send_static_file('js/voice-chat.js')
 
 @socketio.on('connect')
 def handle_connect():
@@ -180,10 +183,6 @@ def handle_connect():
         'should_interrupt_ai': False,
         'ai_stream_queue': queue.Queue(),
         
-        # WebRTC status
-        'webrtc_connected': False,
-        'webrtc_peer_id': None,
-        
         # Processing flags
         'is_processing': False,
         'pending_user_audio': None
@@ -195,10 +194,11 @@ def handle_connect():
         'csm_available': csm_generator is not None,
         'llm_available': llm_model is not None,
         'client_sample_rate': CLIENT_SAMPLE_RATE,
-        'server_sample_rate': getattr(csm_generator, 'sample_rate', 24000) if csm_generator else 24000,
+        'server_sample_rate': getattr(csm_generator, 'sample_rate', 24000) if csm_generator else 24000
-        'ice_servers': ICE_SERVERS
     })
     
+    emit('ready_for_speech', {'message': 'Ready to start conversation'})
+
 @socketio.on('disconnect')
 def handle_disconnect():
     """Handle client disconnection"""
@@ -341,10 +341,10 @@ def on_speech_started(session_id):
     # If AI is speaking, we need to interrupt it
     if session['is_ai_speaking']:
         session['should_interrupt_ai'] = True
-        emit('ai_interrupted_by_user', room=session_id)
+        socketio.emit('ai_interrupted_by_user', room=session_id)
     
     # Notify client that we detected speech
-    emit('user_speech_start', room=session_id)
+    socketio.emit('user_speech_start', room=session_id)
 
 def on_speech_ended(session_id):
     """Handle end of user speech segment"""
@@ -399,12 +399,12 @@ def on_speech_ended(session_id):
         ).start()
         
         # Notify client that processing has started
-        emit('processing_speech', room=session_id)
+        socketio.emit('processing_speech', room=session_id)
     
     except Exception as e:
         print(f"Error preparing audio: {e}")
         session['is_processing'] = False
-        emit('error', {'message': f'Error processing audio: {str(e)}'}, room=session_id)
+        socketio.emit('error', {'message': f'Error processing audio: {str(e)}'}, room=session_id)
 
 def process_user_utterance(session_id, audio_path, audio_tensor):
     """Process user utterance, transcribe and generate response"""
@@ -427,7 +427,7 @@ def process_user_utterance(session_id, audio_path, audio_tensor):
         
         # Check if we got meaningful text
         if not user_text or len(user_text.strip()) < 2:
-            emit('no_speech_detected', room=session_id)
+            socketio.emit('no_speech_detected', room=session_id)  # CHANGED: emit → socketio.emit
             session['is_processing'] = False
             return
         
@@ -448,13 +448,13 @@ def process_user_utterance(session_id, audio_path, audio_tensor):
         })
         
         # Send transcription to client
-        emit('transcription', {'text': user_text}, room=session_id)
+        socketio.emit('transcription', {'text': user_text}, room=session_id)  # CHANGED: emit → socketio.emit
         
         # Generate AI response
         ai_response = generate_ai_response(user_text, session_id)
         
         # Send text response to client
-        emit('ai_response_text', {'text': ai_response}, room=session_id)
+        socketio.emit('ai_response_text', {'text': ai_response}, room=session_id)  # CHANGED: emit → socketio.emit
         
         # Update conversation history
         session['conversation_history'].append({
@@ -476,7 +476,7 @@ def process_user_utterance(session_id, audio_path, audio_tensor):
     
     except Exception as e:
         print(f"Error processing utterance: {e}")
-        emit('error', {'message': f'Error: {str(e)}'}, room=session_id)
+        socketio.emit('error', {'message': f'Error: {str(e)}'}, room=session_id)  # CHANGED: emit → socketio.emit
     
     finally:
         # Clear processing flag
@@ -541,10 +541,12 @@ def generate_ai_response(user_text, session_id):
             inputs = llm_tokenizer(prompt, return_tensors="pt").to(device)
             output = llm_model.generate(
                 inputs.input_ids,
+                attention_mask=inputs.attention_mask,  # Add attention mask
                 max_new_tokens=100,  # Keep responses shorter for voice
                 temperature=0.7,
                 top_p=0.9,
-                do_sample=True
+                do_sample=True,
+                pad_token_id=llm_tokenizer.eos_token_id  # Explicitly set pad_token_id
             )
             
             response = llm_tokenizer.decode(output[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
@@ -587,7 +589,7 @@ def stream_ai_response(text, session_id):
     
     try:
         # Signal start of AI speech
-        emit('ai_speech_start', room=session_id)
+        socketio.emit('ai_speech_start', room=session_id)  # CHANGED: emit → socketio.emit
         
         # Use the last few conversation segments as context (up to 4)
         context_segments = session['segments'][-4:] if len(session['segments']) > 4 else session['segments']
@@ -643,15 +645,15 @@ def stream_ai_response(text, session_id):
         if session_id in user_sessions:
             session['is_ai_speaking'] = False
             session['is_turn_active'] = False  # End conversation turn
-            socketio.emit('ai_speech_end', room=session_id)
+            socketio.emit('ai_speech_end', room=session_id)  # CHANGED: emit → socketio.emit
     
     except Exception as e:
         print(f"Error streaming AI response: {e}")
         if session_id in user_sessions:
             session['is_ai_speaking'] = False
             session['is_turn_active'] = False
-            socketio.emit('error', {'message': f'Error generating audio: {str(e)}'}, room=session_id)
+            socketio.emit('error', {'message': f'Error generating audio: {str(e)}'}, room=session_id)  # CHANGED: emit → socketio.emit
-            socketio.emit('ai_speech_end', room=session_id)
+            socketio.emit('ai_speech_end', room=session_id)  # CHANGED: emit → socketio.emit
 
 @socketio.on('interrupt_ai')
 def handle_interrupt():

README.md

@@ -1 +1,3 @@
 # HooHacks-12
+
+Link to graph: https://docs.google.com/drawings/d/1kRQvTaMHf-dSycMcfUhGtug4g9vPiEZEIeLcZqWd6Nc/edit