diff --git a/Backend/index.html b/Backend/index.html
index 9fcc887..b3b06b7 100644
--- a/Backend/index.html
+++ b/Backend/index.html
@@ -274,7 +274,6 @@
const whisperStatus = document.getElementById('whisper-status');
const csmStatus = document.getElementById('csm-status');
const llmStatus = document.getElementById('llm-status');
- const webrtcStatus = document.getElementById('webrtc-status');
const micAnimation = document.getElementById('mic-animation');
const loadingDiv = document.getElementById('loading');
const loadingText = document.getElementById('loading-text');
@@ -286,14 +285,7 @@
let isAiSpeaking = false;
let audioContext;
let mediaStream;
- let audioRecorder;
let audioProcessor;
- const audioChunks = [];
-
- // WebRTC variables
- let peerConnection;
- let dataChannel;
- let hasActiveConnection = false;
// Audio playback
let audioQueue = [];
@@ -302,7 +294,6 @@
// Configuration variables
let serverSampleRate = 24000;
let clientSampleRate = 44100;
- let iceServers = [];
// Initialize the application
initApp();
@@ -329,7 +320,6 @@
updateConnectionStatus('disconnected');
isConnected = false;
cleanupAudio();
- cleanupWebRTC();
});
socket.on('session_ready', (data) => {
@@ -337,11 +327,13 @@
updateModelStatus(data);
clientSampleRate = data.client_sample_rate;
serverSampleRate = data.server_sample_rate;
- iceServers = data.ice_servers;
- // Initialize WebRTC if models are available
- if (data.whisper_available && data.llm_available) {
- initializeWebRTC();
+ // Enable start button if models are available
+ if (data.whisper_available && data.csm_available) {
+ startButton.disabled = false;
+ addInfoMessage('Ready for conversation. Click "Start Listening" to begin.');
+ } else {
+ addInfoMessage('Some models are not available. Voice chat might not work properly.');
}
});
@@ -351,10 +343,6 @@
addInfoMessage('Ready for conversation. Click "Start Listening" to begin.');
});
- socket.on('webrtc_signal', (data) => {
- handleWebRTCSignal(data);
- });
-
socket.on('transcription', (data) => {
console.log('Transcription:', data);
addUserMessage(data.text);
@@ -460,98 +448,6 @@
llmStatus.style.color = data.llm_available ? 'green' : 'red';
}
- // Initialize WebRTC connection
- function initializeWebRTC() {
- if (!isConnected) return;
-
- const configuration = {
- iceServers: iceServers
- };
-
- peerConnection = new RTCPeerConnection(configuration);
-
- // Create data channel for WebRTC communication
- dataChannel = peerConnection.createDataChannel('audioData', {
- ordered: true
- });
-
- dataChannel.onopen = () => {
- console.log('WebRTC data channel open');
- hasActiveConnection = true;
- webrtcStatus.textContent = 'Connected';
- webrtcStatus.style.color = 'green';
- socket.emit('webrtc_connected', { status: 'connected' });
- };
-
- dataChannel.onclose = () => {
- console.log('WebRTC data channel closed');
- hasActiveConnection = false;
- webrtcStatus.textContent = 'Disconnected';
- webrtcStatus.style.color = 'red';
- };
-
- // Handle ICE candidates
- peerConnection.onicecandidate = (event) => {
- if (event.candidate) {
- socket.emit('webrtc_signal', {
- type: 'ice_candidate',
- candidate: event.candidate
- });
- }
- };
-
- // Log ICE connection state changes
- peerConnection.oniceconnectionstatechange = () => {
- console.log('ICE connection state:', peerConnection.iceConnectionState);
- };
-
- // Create offer
- peerConnection.createOffer()
- .then(offer => peerConnection.setLocalDescription(offer))
- .then(() => {
- socket.emit('webrtc_signal', {
- type: 'offer',
- sdp: peerConnection.localDescription
- });
- })
- .catch(error => {
- console.error('Error creating WebRTC offer:', error);
- webrtcStatus.textContent = 'Failed to Connect';
- webrtcStatus.style.color = 'red';
- });
- }
-
- // Handle WebRTC signals from the server
- function handleWebRTCSignal(data) {
- if (!peerConnection) return;
-
- if (data.type === 'answer') {
- peerConnection.setRemoteDescription(new RTCSessionDescription(data.sdp))
- .catch(error => console.error('Error setting remote description:', error));
- }
- else if (data.type === 'ice_candidate') {
- peerConnection.addIceCandidate(new RTCIceCandidate(data.candidate))
- .catch(error => console.error('Error adding ICE candidate:', error));
- }
- }
-
- // Clean up WebRTC connection
- function cleanupWebRTC() {
- if (dataChannel) {
- dataChannel.close();
- }
-
- if (peerConnection) {
- peerConnection.close();
- }
-
- dataChannel = null;
- peerConnection = null;
- hasActiveConnection = false;
- webrtcStatus.textContent = 'Not Connected';
- webrtcStatus.style.color = 'red';
- }
-
// Toggle audio listening
function toggleListening() {
if (isListening) {
@@ -648,8 +544,6 @@
if (audioContext && audioContext.state !== 'closed') {
audioContext.close().catch(error => console.error('Error closing AudioContext:', error));
}
-
- audioChunks.length = 0;
}
// Convert Float32Array to Int16Array for sending to server
@@ -669,7 +563,7 @@
// Convert to base64 for transmission
const base64Audio = arrayBufferToBase64(audioData.buffer);
- // Send via Socket.IO (could use WebRTC's DataChannel for lower latency in production)
+ // Send via Socket.IO
socket.emit('audio_stream', { audio: base64Audio });
}
diff --git a/Backend/server.py b/Backend/server.py
index 63a5846..1e87780 100644
--- a/Backend/server.py
+++ b/Backend/server.py
@@ -152,7 +152,10 @@ def index():
"""Serve the main interface"""
return render_template('index.html')
-
+@app.route('/static/js/voice-chat.js')
+def serve_voice_chat_js():
+ """Serve the JavaScript file"""
+ return app.send_static_file('js/voice-chat.js')
@socketio.on('connect')
def handle_connect():
@@ -180,10 +183,6 @@ def handle_connect():
'should_interrupt_ai': False,
'ai_stream_queue': queue.Queue(),
- # WebRTC status
- 'webrtc_connected': False,
- 'webrtc_peer_id': None,
-
# Processing flags
'is_processing': False,
'pending_user_audio': None
@@ -195,9 +194,10 @@ def handle_connect():
'csm_available': csm_generator is not None,
'llm_available': llm_model is not None,
'client_sample_rate': CLIENT_SAMPLE_RATE,
- 'server_sample_rate': getattr(csm_generator, 'sample_rate', 24000) if csm_generator else 24000,
- 'ice_servers': ICE_SERVERS
+ 'server_sample_rate': getattr(csm_generator, 'sample_rate', 24000) if csm_generator else 24000
})
+
+ emit('ready_for_speech', {'message': 'Ready to start conversation'})
@socketio.on('disconnect')
def handle_disconnect():
@@ -341,10 +341,10 @@ def on_speech_started(session_id):
# If AI is speaking, we need to interrupt it
if session['is_ai_speaking']:
session['should_interrupt_ai'] = True
- emit('ai_interrupted_by_user', room=session_id)
+ socketio.emit('ai_interrupted_by_user', room=session_id)
# Notify client that we detected speech
- emit('user_speech_start', room=session_id)
+ socketio.emit('user_speech_start', room=session_id)
def on_speech_ended(session_id):
"""Handle end of user speech segment"""
@@ -399,12 +399,12 @@ def on_speech_ended(session_id):
).start()
# Notify client that processing has started
- emit('processing_speech', room=session_id)
+ socketio.emit('processing_speech', room=session_id)
except Exception as e:
print(f"Error preparing audio: {e}")
session['is_processing'] = False
- emit('error', {'message': f'Error processing audio: {str(e)}'}, room=session_id)
+ socketio.emit('error', {'message': f'Error processing audio: {str(e)}'}, room=session_id)
def process_user_utterance(session_id, audio_path, audio_tensor):
"""Process user utterance, transcribe and generate response"""
@@ -427,7 +427,7 @@ def process_user_utterance(session_id, audio_path, audio_tensor):
# Check if we got meaningful text
if not user_text or len(user_text.strip()) < 2:
- emit('no_speech_detected', room=session_id)
+ socketio.emit('no_speech_detected', room=session_id) # CHANGED: emit → socketio.emit
session['is_processing'] = False
return
@@ -448,13 +448,13 @@ def process_user_utterance(session_id, audio_path, audio_tensor):
})
# Send transcription to client
- emit('transcription', {'text': user_text}, room=session_id)
+ socketio.emit('transcription', {'text': user_text}, room=session_id) # CHANGED: emit → socketio.emit
# Generate AI response
ai_response = generate_ai_response(user_text, session_id)
# Send text response to client
- emit('ai_response_text', {'text': ai_response}, room=session_id)
+ socketio.emit('ai_response_text', {'text': ai_response}, room=session_id) # CHANGED: emit → socketio.emit
# Update conversation history
session['conversation_history'].append({
@@ -476,7 +476,7 @@ def process_user_utterance(session_id, audio_path, audio_tensor):
except Exception as e:
print(f"Error processing utterance: {e}")
- emit('error', {'message': f'Error: {str(e)}'}, room=session_id)
+ socketio.emit('error', {'message': f'Error: {str(e)}'}, room=session_id) # CHANGED: emit → socketio.emit
finally:
# Clear processing flag
@@ -540,11 +540,13 @@ def generate_ai_response(user_text, session_id):
# Generate response
inputs = llm_tokenizer(prompt, return_tensors="pt").to(device)
output = llm_model.generate(
- inputs.input_ids,
+ inputs.input_ids,
+ attention_mask=inputs.attention_mask, # Add attention mask
max_new_tokens=100, # Keep responses shorter for voice
temperature=0.7,
top_p=0.9,
- do_sample=True
+ do_sample=True,
+ pad_token_id=llm_tokenizer.eos_token_id # Explicitly set pad_token_id
)
response = llm_tokenizer.decode(output[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
@@ -587,7 +589,7 @@ def stream_ai_response(text, session_id):
try:
# Signal start of AI speech
- emit('ai_speech_start', room=session_id)
+ socketio.emit('ai_speech_start', room=session_id) # CHANGED: emit → socketio.emit
# Use the last few conversation segments as context (up to 4)
context_segments = session['segments'][-4:] if len(session['segments']) > 4 else session['segments']
@@ -643,15 +645,15 @@ def stream_ai_response(text, session_id):
if session_id in user_sessions:
session['is_ai_speaking'] = False
session['is_turn_active'] = False # End conversation turn
- socketio.emit('ai_speech_end', room=session_id)
+ socketio.emit('ai_speech_end', room=session_id) # CHANGED: emit → socketio.emit
except Exception as e:
print(f"Error streaming AI response: {e}")
if session_id in user_sessions:
session['is_ai_speaking'] = False
session['is_turn_active'] = False
- socketio.emit('error', {'message': f'Error generating audio: {str(e)}'}, room=session_id)
- socketio.emit('ai_speech_end', room=session_id)
+ socketio.emit('error', {'message': f'Error generating audio: {str(e)}'}, room=session_id) # CHANGED: emit → socketio.emit
+ socketio.emit('ai_speech_end', room=session_id) # CHANGED: emit → socketio.emit
@socketio.on('interrupt_ai')
def handle_interrupt():
diff --git a/README.md b/README.md
index bf49ea8..d757f43 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,3 @@
-# HooHacks-12
\ No newline at end of file
+# HooHacks-12
+
+Link to graph: https://docs.google.com/drawings/d/1kRQvTaMHf-dSycMcfUhGtug4g9vPiEZEIeLcZqWd6Nc/edit
\ No newline at end of file