feat: activity video track (pulsing orb) for voice sessions

- ActivityVideoPublisher renders animated orb on 160x120 canvas
- Integrated into both agent.py and voice.py
- Updates confluence-collab submodule
This commit is contained in:
Christian Gick
2026-03-06 15:58:51 +00:00
parent 947699c988
commit efb976a27c
4 changed files with 199 additions and 1 deletions

View File

@@ -1,3 +1,4 @@
import asyncio
import os
import json
import base64
@@ -10,6 +11,7 @@ from livekit.plugins import openai as lk_openai, elevenlabs, silero
import livekit.rtc as rtc
from e2ee_patch import KDF_HKDF
from activity_video import ActivityVideoPublisher
logger = logging.getLogger("matrix-ai-agent")
logging.basicConfig(level=logging.DEBUG)
@@ -103,6 +105,13 @@ async def entrypoint(ctx: JobContext):
logger.info("Connected to room, local identity: %s", ctx.room.local_participant.identity)
logger.info("Remote participants: %s", list(ctx.room.remote_participants.keys()))
# Publish activity video track (animated waveform bars)
activity_video = ActivityVideoPublisher()
video_track = rtc.LocalVideoTrack.create_video_track("activity", activity_video.source)
await ctx.room.local_participant.publish_track(video_track)
activity_task = asyncio.create_task(activity_video.run())
logger.info("Activity video track published")
model = os.environ.get("LITELLM_MODEL", "claude-sonnet")
voice_id = os.environ.get("ELEVENLABS_VOICE_ID", "21m00Tcm4TlvDq8ikWAM")
@@ -120,6 +129,10 @@ async def entrypoint(ctx: JobContext):
vad=ctx.proc.userdata["vad"],
)
@session.on("agent_state_changed")
def on_state_changed(ev):
activity_video.set_state(ev.new_state)
@session.on("user_speech_committed")
def on_speech(msg):
logger.info("USER_SPEECH_COMMITTED: %s", msg.text_content)