feat: activity video track (pulsing orb) for voice sessions
- ActivityVideoPublisher renders animated orb on 160x120 canvas - Integrated into both agent.py and voice.py - Updates confluence-collab submodule
This commit is contained in:
13
agent.py
13
agent.py
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import os
|
||||
import json
|
||||
import base64
|
||||
@@ -10,6 +11,7 @@ from livekit.plugins import openai as lk_openai, elevenlabs, silero
|
||||
import livekit.rtc as rtc
|
||||
|
||||
from e2ee_patch import KDF_HKDF
|
||||
from activity_video import ActivityVideoPublisher
|
||||
|
||||
logger = logging.getLogger("matrix-ai-agent")
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
@@ -103,6 +105,13 @@ async def entrypoint(ctx: JobContext):
|
||||
logger.info("Connected to room, local identity: %s", ctx.room.local_participant.identity)
|
||||
logger.info("Remote participants: %s", list(ctx.room.remote_participants.keys()))
|
||||
|
||||
# Publish activity video track (animated waveform bars)
|
||||
activity_video = ActivityVideoPublisher()
|
||||
video_track = rtc.LocalVideoTrack.create_video_track("activity", activity_video.source)
|
||||
await ctx.room.local_participant.publish_track(video_track)
|
||||
activity_task = asyncio.create_task(activity_video.run())
|
||||
logger.info("Activity video track published")
|
||||
|
||||
model = os.environ.get("LITELLM_MODEL", "claude-sonnet")
|
||||
voice_id = os.environ.get("ELEVENLABS_VOICE_ID", "21m00Tcm4TlvDq8ikWAM")
|
||||
|
||||
@@ -120,6 +129,10 @@ async def entrypoint(ctx: JobContext):
|
||||
vad=ctx.proc.userdata["vad"],
|
||||
)
|
||||
|
||||
@session.on("agent_state_changed")
|
||||
def on_state_changed(ev):
|
||||
activity_video.set_state(ev.new_state)
|
||||
|
||||
@session.on("user_speech_committed")
|
||||
def on_speech(msg):
|
||||
logger.info("USER_SPEECH_COMMITTED: %s", msg.text_content)
|
||||
|
||||
Reference in New Issue
Block a user