Backend Server Update
This commit is contained in:
@@ -10,60 +10,113 @@
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
background-color: #f9f9f9;
|
||||
}
|
||||
.conversation {
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 8px;
|
||||
padding: 15px;
|
||||
height: 300px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 12px;
|
||||
padding: 20px;
|
||||
height: 400px;
|
||||
overflow-y: auto;
|
||||
margin-bottom: 15px;
|
||||
margin-bottom: 20px;
|
||||
background-color: white;
|
||||
box-shadow: 0 2px 10px rgba(0,0,0,0.05);
|
||||
}
|
||||
.message {
|
||||
margin-bottom: 10px;
|
||||
padding: 8px;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 15px;
|
||||
padding: 12px;
|
||||
border-radius: 12px;
|
||||
max-width: 80%;
|
||||
line-height: 1.4;
|
||||
}
|
||||
.user {
|
||||
background-color: #e3f2fd;
|
||||
text-align: right;
|
||||
margin-left: auto;
|
||||
border-bottom-right-radius: 4px;
|
||||
}
|
||||
.ai {
|
||||
background-color: #f1f1f1;
|
||||
margin-right: auto;
|
||||
border-bottom-left-radius: 4px;
|
||||
}
|
||||
.system {
|
||||
background-color: #f8f9fa;
|
||||
font-style: italic;
|
||||
text-align: center;
|
||||
font-size: 0.9em;
|
||||
color: #666;
|
||||
padding: 8px;
|
||||
margin: 10px auto;
|
||||
max-width: 90%;
|
||||
}
|
||||
.controls {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
.input-row {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
}
|
||||
input[type="text"] {
|
||||
flex-grow: 1;
|
||||
padding: 8px;
|
||||
border-radius: 4px;
|
||||
border: 1px solid #ccc;
|
||||
gap: 15px;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
}
|
||||
button {
|
||||
padding: 8px 16px;
|
||||
border-radius: 4px;
|
||||
padding: 12px 24px;
|
||||
border-radius: 24px;
|
||||
border: none;
|
||||
background-color: #4CAF50;
|
||||
color: white;
|
||||
cursor: pointer;
|
||||
font-weight: bold;
|
||||
transition: all 0.2s ease;
|
||||
box-shadow: 0 2px 5px rgba(0,0,0,0.1);
|
||||
}
|
||||
button:hover {
|
||||
background-color: #45a049;
|
||||
box-shadow: 0 4px 8px rgba(0,0,0,0.15);
|
||||
}
|
||||
.recording {
|
||||
background-color: #f44336;
|
||||
animation: pulse 1.5s infinite;
|
||||
}
|
||||
.processing {
|
||||
background-color: #FFA500;
|
||||
}
|
||||
select {
|
||||
padding: 8px;
|
||||
border-radius: 4px;
|
||||
border: 1px solid #ccc;
|
||||
padding: 10px;
|
||||
border-radius: 24px;
|
||||
border: 1px solid #ddd;
|
||||
background-color: white;
|
||||
}
|
||||
.transcript {
|
||||
font-style: italic;
|
||||
color: #666;
|
||||
margin-top: 5px;
|
||||
}
|
||||
@keyframes pulse {
|
||||
0% { opacity: 1; }
|
||||
50% { opacity: 0.7; }
|
||||
100% { opacity: 1; }
|
||||
}
|
||||
.status-indicator {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
margin-top: 10px;
|
||||
gap: 5px;
|
||||
}
|
||||
.status-dot {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 50%;
|
||||
background-color: #ccc;
|
||||
}
|
||||
.status-dot.active {
|
||||
background-color: #4CAF50;
|
||||
}
|
||||
.status-text {
|
||||
font-size: 0.9em;
|
||||
color: #666;
|
||||
}
|
||||
audio {
|
||||
width: 100%;
|
||||
margin-top: 5px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
@@ -72,30 +125,25 @@
|
||||
<div class="conversation" id="conversation"></div>
|
||||
|
||||
<div class="controls">
|
||||
<div class="input-row">
|
||||
<input type="text" id="textInput" placeholder="Type your message...">
|
||||
<select id="speakerSelect">
|
||||
<option value="0">Speaker 0</option>
|
||||
<option value="1">Speaker 1</option>
|
||||
</select>
|
||||
<button id="sendText">Send</button>
|
||||
</div>
|
||||
|
||||
<div class="input-row">
|
||||
<button id="recordAudio">Record Audio</button>
|
||||
<button id="clearContext">Clear Context</button>
|
||||
</div>
|
||||
<select id="speakerSelect">
|
||||
<option value="0">Speaker 0</option>
|
||||
<option value="1">Speaker 1</option>
|
||||
</select>
|
||||
<button id="streamButton">Start Conversation</button>
|
||||
<button id="clearButton">Clear Chat</button>
|
||||
</div>
|
||||
|
||||
<div class="status-indicator">
|
||||
<div class="status-dot" id="statusDot"></div>
|
||||
<div class="status-text" id="statusText">Not connected</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Variables
|
||||
let ws;
|
||||
let mediaRecorder;
|
||||
let audioChunks = [];
|
||||
let isRecording = false;
|
||||
let audioContext;
|
||||
let streamProcessor;
|
||||
let isStreaming = false;
|
||||
let streamButton;
|
||||
let isSpeaking = false;
|
||||
let silenceTimer = null;
|
||||
let energyWindow = [];
|
||||
@@ -105,24 +153,20 @@
|
||||
|
||||
// DOM elements
|
||||
const conversationEl = document.getElementById('conversation');
|
||||
const textInputEl = document.getElementById('textInput');
|
||||
const speakerSelectEl = document.getElementById('speakerSelect');
|
||||
const sendTextBtn = document.getElementById('sendText');
|
||||
const recordAudioBtn = document.getElementById('recordAudio');
|
||||
const clearContextBtn = document.getElementById('clearContext');
|
||||
const streamButton = document.getElementById('streamButton');
|
||||
const clearButton = document.getElementById('clearButton');
|
||||
const statusDot = document.getElementById('statusDot');
|
||||
const statusText = document.getElementById('statusText');
|
||||
|
||||
// Add streaming button to the input row
|
||||
// Initialize on page load
|
||||
window.addEventListener('load', () => {
|
||||
const inputRow = document.querySelector('.input-row:nth-child(2)');
|
||||
streamButton = document.createElement('button');
|
||||
streamButton.id = 'streamAudio';
|
||||
streamButton.textContent = 'Start Streaming';
|
||||
streamButton.addEventListener('click', toggleStreaming);
|
||||
inputRow.appendChild(streamButton);
|
||||
|
||||
connectWebSocket();
|
||||
setupRecording();
|
||||
setupAudioContext();
|
||||
|
||||
// Event listeners
|
||||
streamButton.addEventListener('click', toggleStreaming);
|
||||
clearButton.addEventListener('click', clearConversation);
|
||||
});
|
||||
|
||||
// Setup audio context for streaming
|
||||
@@ -136,8 +180,68 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Toggle audio streaming
|
||||
async function toggleStreaming() {
|
||||
// Connect to WebSocket server
|
||||
function connectWebSocket() {
|
||||
const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
const wsUrl = `${wsProtocol}//${window.location.hostname}:8000/ws`;
|
||||
|
||||
ws = new WebSocket(wsUrl);
|
||||
|
||||
ws.onopen = () => {
|
||||
console.log('WebSocket connected');
|
||||
statusDot.classList.add('active');
|
||||
statusText.textContent = 'Connected';
|
||||
addSystemMessage('Connected to server');
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
const response = JSON.parse(event.data);
|
||||
console.log('Received:', response);
|
||||
|
||||
if (response.type === 'audio_response') {
|
||||
// Play audio response
|
||||
const audio = new Audio(response.audio);
|
||||
audio.play();
|
||||
|
||||
// Add message to conversation
|
||||
addAIMessage(response.text || 'AI response', response.audio);
|
||||
|
||||
// Reset to speaking state after AI response
|
||||
if (isStreaming) {
|
||||
streamButton.textContent = 'Listening...';
|
||||
streamButton.style.backgroundColor = '#f44336'; // Back to red
|
||||
streamButton.classList.add('recording');
|
||||
isSpeaking = false; // Reset speaking state
|
||||
}
|
||||
} else if (response.type === 'error') {
|
||||
addSystemMessage(`Error: ${response.message}`);
|
||||
} else if (response.type === 'context_updated') {
|
||||
addSystemMessage(response.message);
|
||||
} else if (response.type === 'streaming_status') {
|
||||
addSystemMessage(`Streaming ${response.status}`);
|
||||
} else if (response.type === 'transcription') {
|
||||
addUserTranscription(response.text);
|
||||
}
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
console.log('WebSocket disconnected');
|
||||
statusDot.classList.remove('active');
|
||||
statusText.textContent = 'Disconnected';
|
||||
addSystemMessage('Disconnected from server. Reconnecting...');
|
||||
setTimeout(connectWebSocket, 3000);
|
||||
};
|
||||
|
||||
ws.onerror = (error) => {
|
||||
console.error('WebSocket error:', error);
|
||||
statusDot.classList.remove('active');
|
||||
statusText.textContent = 'Error';
|
||||
addSystemMessage('Connection error');
|
||||
};
|
||||
}
|
||||
|
||||
// Toggle streaming
|
||||
function toggleStreaming() {
|
||||
if (isStreaming) {
|
||||
stopStreaming();
|
||||
} else {
|
||||
@@ -145,7 +249,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Start audio streaming with silence detection
|
||||
// Start streaming
|
||||
async function startStreaming() {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
@@ -155,7 +259,7 @@
|
||||
isSpeaking = false;
|
||||
energyWindow = [];
|
||||
|
||||
streamButton.textContent = 'Speaking...';
|
||||
streamButton.textContent = 'Listening...';
|
||||
streamButton.classList.add('recording');
|
||||
|
||||
// Create audio processor node
|
||||
@@ -186,13 +290,13 @@
|
||||
source.connect(streamProcessor);
|
||||
streamProcessor.connect(audioContext.destination);
|
||||
|
||||
addSystemMessage('Audio streaming started - speak naturally and pause when finished');
|
||||
addSystemMessage('Listening - speak naturally and pause when finished');
|
||||
|
||||
} catch (err) {
|
||||
console.error('Error starting audio stream:', err);
|
||||
addSystemMessage(`Streaming error: ${err.message}`);
|
||||
addSystemMessage(`Microphone error: ${err.message}`);
|
||||
isStreaming = false;
|
||||
streamButton.textContent = 'Start Streaming';
|
||||
streamButton.textContent = 'Start Conversation';
|
||||
streamButton.classList.remove('recording');
|
||||
}
|
||||
}
|
||||
@@ -228,15 +332,17 @@
|
||||
silenceTimer = setTimeout(() => {
|
||||
// Silence persisted long enough
|
||||
streamButton.textContent = 'Processing...';
|
||||
streamButton.style.backgroundColor = '#FFA500'; // Orange
|
||||
streamButton.classList.remove('recording');
|
||||
streamButton.classList.add('processing');
|
||||
addSystemMessage('Detected pause in speech, processing response...');
|
||||
}, CLIENT_SILENCE_DURATION_MS);
|
||||
}
|
||||
} else if (!isSpeaking && !isSilent) {
|
||||
// Transition from silence to speaking
|
||||
isSpeaking = true;
|
||||
streamButton.textContent = 'Speaking...';
|
||||
streamButton.style.backgroundColor = '#f44336'; // Red
|
||||
streamButton.textContent = 'Listening...';
|
||||
streamButton.classList.add('recording');
|
||||
streamButton.classList.remove('processing');
|
||||
|
||||
// Clear any pending silence timer
|
||||
if (silenceTimer) {
|
||||
@@ -276,7 +382,7 @@
|
||||
reader.readAsDataURL(wavData);
|
||||
}
|
||||
|
||||
// Stop audio streaming
|
||||
// Stop streaming
|
||||
function stopStreaming() {
|
||||
if (streamProcessor) {
|
||||
streamProcessor.disconnect();
|
||||
@@ -293,11 +399,11 @@
|
||||
isSpeaking = false;
|
||||
energyWindow = [];
|
||||
|
||||
streamButton.textContent = 'Start Streaming';
|
||||
streamButton.classList.remove('recording');
|
||||
streamButton.textContent = 'Start Conversation';
|
||||
streamButton.classList.remove('recording', 'processing');
|
||||
streamButton.style.backgroundColor = ''; // Reset to default
|
||||
|
||||
addSystemMessage('Audio streaming stopped');
|
||||
addSystemMessage('Conversation paused');
|
||||
|
||||
// Send stop streaming signal to server
|
||||
ws.send(JSON.stringify({
|
||||
@@ -306,6 +412,18 @@
|
||||
}));
|
||||
}
|
||||
|
||||
// Clear conversation
|
||||
function clearConversation() {
|
||||
// Clear conversation history
|
||||
ws.send(JSON.stringify({
|
||||
action: 'clear_context'
|
||||
}));
|
||||
|
||||
// Clear the UI
|
||||
conversationEl.innerHTML = '';
|
||||
addSystemMessage('Conversation cleared');
|
||||
}
|
||||
|
||||
// Downsample audio buffer to target sample rate
|
||||
function downsampleBuffer(buffer, sampleRate, targetSampleRate) {
|
||||
if (targetSampleRate === sampleRate) {
|
||||
@@ -376,212 +494,49 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Connect to WebSocket
|
||||
function connectWebSocket() {
|
||||
const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
const wsUrl = `${wsProtocol}//${window.location.hostname}:8000/ws`;
|
||||
// Message display functions
|
||||
function addUserTranscription(text) {
|
||||
// Find if there's already a pending user message
|
||||
let pendingMessage = document.querySelector('.message.user.pending');
|
||||
|
||||
ws = new WebSocket(wsUrl);
|
||||
if (!pendingMessage) {
|
||||
// Create a new message
|
||||
pendingMessage = document.createElement('div');
|
||||
pendingMessage.classList.add('message', 'user', 'pending');
|
||||
conversationEl.appendChild(pendingMessage);
|
||||
}
|
||||
|
||||
ws.onopen = () => {
|
||||
console.log('WebSocket connected');
|
||||
addSystemMessage('Connected to server');
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
const response = JSON.parse(event.data);
|
||||
console.log('Received:', response);
|
||||
|
||||
if (response.type === 'audio_response') {
|
||||
// Play audio response
|
||||
const audio = new Audio(response.audio);
|
||||
audio.play();
|
||||
|
||||
// Add message to conversation
|
||||
addAIMessage(response.audio);
|
||||
|
||||
// Reset the streaming button if we're still in streaming mode
|
||||
if (isStreaming) {
|
||||
streamButton.textContent = 'Speaking...';
|
||||
streamButton.style.backgroundColor = '#f44336'; // Back to red
|
||||
isSpeaking = false; // Reset speaking state
|
||||
}
|
||||
} else if (response.type === 'error') {
|
||||
addSystemMessage(`Error: ${response.message}`);
|
||||
} else if (response.type === 'context_updated') {
|
||||
addSystemMessage(response.message);
|
||||
} else if (response.type === 'streaming_status') {
|
||||
addSystemMessage(`Streaming ${response.status}`);
|
||||
}
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
console.log('WebSocket disconnected');
|
||||
addSystemMessage('Disconnected from server. Reconnecting...');
|
||||
setTimeout(connectWebSocket, 3000);
|
||||
};
|
||||
|
||||
ws.onerror = (error) => {
|
||||
console.error('WebSocket error:', error);
|
||||
addSystemMessage('Connection error');
|
||||
};
|
||||
}
|
||||
|
||||
// Add message to conversation
|
||||
function addUserMessage(text) {
|
||||
const messageEl = document.createElement('div');
|
||||
messageEl.classList.add('message', 'user');
|
||||
messageEl.textContent = text;
|
||||
conversationEl.appendChild(messageEl);
|
||||
pendingMessage.textContent = text;
|
||||
pendingMessage.classList.remove('pending');
|
||||
conversationEl.scrollTop = conversationEl.scrollHeight;
|
||||
}
|
||||
|
||||
function addAIMessage(audioSrc) {
|
||||
function addAIMessage(text, audioSrc) {
|
||||
const messageEl = document.createElement('div');
|
||||
messageEl.classList.add('message', 'ai');
|
||||
|
||||
if (text) {
|
||||
const textDiv = document.createElement('div');
|
||||
textDiv.textContent = text;
|
||||
messageEl.appendChild(textDiv);
|
||||
}
|
||||
|
||||
const audioEl = document.createElement('audio');
|
||||
audioEl.controls = true;
|
||||
audioEl.src = audioSrc;
|
||||
|
||||
messageEl.appendChild(audioEl);
|
||||
|
||||
conversationEl.appendChild(messageEl);
|
||||
conversationEl.scrollTop = conversationEl.scrollHeight;
|
||||
}
|
||||
|
||||
function addSystemMessage(text) {
|
||||
const messageEl = document.createElement('div');
|
||||
messageEl.classList.add('message');
|
||||
messageEl.classList.add('message', 'system');
|
||||
messageEl.textContent = text;
|
||||
conversationEl.appendChild(messageEl);
|
||||
conversationEl.scrollTop = conversationEl.scrollHeight;
|
||||
}
|
||||
|
||||
// Send text for audio generation
|
||||
function sendTextForGeneration() {
|
||||
const text = textInputEl.value.trim();
|
||||
const speaker = parseInt(speakerSelectEl.value);
|
||||
|
||||
if (!text) return;
|
||||
|
||||
addUserMessage(text);
|
||||
textInputEl.value = '';
|
||||
|
||||
const request = {
|
||||
action: 'generate',
|
||||
text: text,
|
||||
speaker: speaker
|
||||
};
|
||||
|
||||
ws.send(JSON.stringify(request));
|
||||
}
|
||||
|
||||
// Audio recording functions
|
||||
async function setupRecording() {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
|
||||
mediaRecorder = new MediaRecorder(stream);
|
||||
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
audioChunks.push(event.data);
|
||||
}
|
||||
};
|
||||
|
||||
mediaRecorder.onstop = async () => {
|
||||
const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
|
||||
const audioUrl = URL.createObjectURL(audioBlob);
|
||||
|
||||
// Add audio to conversation
|
||||
addUserMessage('Recorded audio:');
|
||||
const messageEl = document.createElement('div');
|
||||
messageEl.classList.add('message', 'user');
|
||||
|
||||
const audioEl = document.createElement('audio');
|
||||
audioEl.controls = true;
|
||||
audioEl.src = audioUrl;
|
||||
|
||||
messageEl.appendChild(audioEl);
|
||||
conversationEl.appendChild(messageEl);
|
||||
|
||||
// Convert to base64
|
||||
const reader = new FileReader();
|
||||
reader.readAsDataURL(audioBlob);
|
||||
reader.onloadend = () => {
|
||||
const base64Audio = reader.result;
|
||||
const text = textInputEl.value.trim() || "Recorded audio";
|
||||
const speaker = parseInt(speakerSelectEl.value);
|
||||
|
||||
// Send to server
|
||||
const request = {
|
||||
action: 'add_to_context',
|
||||
text: text,
|
||||
speaker: speaker,
|
||||
audio: base64Audio
|
||||
};
|
||||
|
||||
ws.send(JSON.stringify(request));
|
||||
textInputEl.value = '';
|
||||
};
|
||||
|
||||
audioChunks = [];
|
||||
recordAudioBtn.textContent = 'Record Audio';
|
||||
recordAudioBtn.classList.remove('recording');
|
||||
};
|
||||
|
||||
console.log('Recording setup completed');
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error('Error setting up recording:', err);
|
||||
addSystemMessage(`Microphone access error: ${err.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function toggleRecording() {
|
||||
if (isRecording) {
|
||||
mediaRecorder.stop();
|
||||
isRecording = false;
|
||||
} else {
|
||||
if (!mediaRecorder) {
|
||||
setupRecording().then(success => {
|
||||
if (success) startRecording();
|
||||
});
|
||||
} else {
|
||||
startRecording();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function startRecording() {
|
||||
audioChunks = [];
|
||||
mediaRecorder.start();
|
||||
isRecording = true;
|
||||
recordAudioBtn.textContent = 'Stop Recording';
|
||||
recordAudioBtn.classList.add('recording');
|
||||
}
|
||||
|
||||
// Event listeners
|
||||
sendTextBtn.addEventListener('click', sendTextForGeneration);
|
||||
|
||||
textInputEl.addEventListener('keypress', (e) => {
|
||||
if (e.key === 'Enter') sendTextForGeneration();
|
||||
});
|
||||
|
||||
recordAudioBtn.addEventListener('click', toggleRecording);
|
||||
|
||||
clearContextBtn.addEventListener('click', () => {
|
||||
ws.send(JSON.stringify({
|
||||
action: 'clear_context'
|
||||
}));
|
||||
});
|
||||
|
||||
// Initialize
|
||||
window.addEventListener('load', () => {
|
||||
connectWebSocket();
|
||||
setupRecording();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user