From fa65fbeb3d26bab88100448727ad65974bad2bc7 Mon Sep 17 00:00:00 2001 From: Christian Gick Date: Sun, 15 Feb 2026 07:30:18 +0200 Subject: [PATCH] feat: Matrix AI voice agent (LiveKit + LiteLLM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bot @ai:agiliton.eu accepts room invites, dispatches LiveKit agent. Agent joins call with STT (Groq Whisper) → LLM (Sonnet) → TTS (ElevenLabs) pipeline, all routed through LiteLLM. CF-1147 Co-Authored-By: Claude Opus 4.6 --- .gitignore | 4 +++ Dockerfile | 6 ++++ agent.py | 58 ++++++++++++++++++++++++++++++++++++ bot.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 13 +++++++++ requirements.txt | 7 +++++ 6 files changed, 161 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 agent.py create mode 100644 bot.py create mode 100644 docker-compose.yml create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fc36f7d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +__pycache__/ +*.pyc +.venv/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bbe8141 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-slim +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . diff --git a/agent.py b/agent.py new file mode 100644 index 0000000..21a0db3 --- /dev/null +++ b/agent.py @@ -0,0 +1,58 @@ +import os +import logging + +from livekit.agents import Agent, AgentSession, AgentServer, JobContext, JobProcess, cli +from livekit.plugins import openai as lk_openai, elevenlabs, silero + +logger = logging.getLogger("matrix-ai-agent") + +LITELLM_URL = os.environ["LITELLM_BASE_URL"] +LITELLM_KEY = os.environ.get("LITELLM_API_KEY", "not-needed") + +SYSTEM_PROMPT = """You are a helpful voice assistant in a Matrix call. +Rules: +- Keep answers SHORT — 1-3 sentences max +- Be direct, no filler words +- If the user wants more detail, they will ask +- Speak naturally as in a conversation""" + +server = AgentServer() + + +def prewarm(proc: JobProcess): + proc.userdata["vad"] = silero.VAD.load() + + +server.setup_fnc = prewarm + + +@server.rtc_session(agent_name="matrix-ai") +async def entrypoint(ctx: JobContext): + model = os.environ.get("LITELLM_MODEL", "claude-sonnet") + voice_id = os.environ.get("ELEVENLABS_VOICE_ID", "21m00Tcm4TlvDq8ikWAM") + + session = AgentSession( + stt=lk_openai.STT( + base_url=LITELLM_URL, + api_key=LITELLM_KEY, + model="whisper", + ), + llm=lk_openai.LLM( + base_url=LITELLM_URL, + api_key=LITELLM_KEY, + model=model, + ), + tts=elevenlabs.TTS( + voice=voice_id, + model="eleven_multilingual_v2", + ), + vad=ctx.proc.userdata["vad"], + ) + + agent = Agent(instructions=SYSTEM_PROMPT) + await session.start(agent=agent, room=ctx.room) + await session.generate_reply(instructions="Greet the user briefly.") + + +if __name__ == "__main__": + cli.run_app(server) diff --git a/bot.py b/bot.py new file mode 100644 index 0000000..e55c0b1 --- /dev/null +++ b/bot.py @@ -0,0 +1,73 @@ +import os +import asyncio +import logging + +from nio import AsyncClient, LoginResponse, InviteMemberEvent +from livekit import api + +logger = logging.getLogger("matrix-ai-bot") + +HOMESERVER = os.environ["MATRIX_HOMESERVER"] +BOT_USER = os.environ["MATRIX_BOT_USER"] +BOT_PASS = os.environ["MATRIX_BOT_PASSWORD"] +LK_URL = os.environ["LIVEKIT_URL"] +LK_KEY = os.environ["LIVEKIT_API_KEY"] +LK_SECRET = os.environ["LIVEKIT_API_SECRET"] +AGENT_NAME = os.environ.get("AGENT_NAME", "matrix-ai") + + +class Bot: + def __init__(self): + self.client = AsyncClient(HOMESERVER, BOT_USER) + self.lkapi = None + self.dispatched_rooms = set() + + async def start(self): + resp = await self.client.login(BOT_PASS, device_name="ai-voice-bot") + if not isinstance(resp, LoginResponse): + logger.error("Login failed: %s", resp) + return + logger.info("Logged in as %s", resp.user_id) + + self.lkapi = api.LiveKitAPI(LK_URL, LK_KEY, LK_SECRET) + self.client.add_event_callback(self.on_invite, InviteMemberEvent) + + await self.client.sync_forever(timeout=30000) + + async def on_invite(self, room, event: InviteMemberEvent): + if event.state_key != BOT_USER: + return + logger.info("Invited to %s", room.room_id) + await self.client.join(room.room_id) + + # LiveKit room name = Matrix room ID (Element Call convention) + lk_room_name = room.room_id + try: + await self.lkapi.agent_dispatch.create_dispatch( + api.CreateAgentDispatchRequest( + agent_name=AGENT_NAME, + room=lk_room_name, + ) + ) + self.dispatched_rooms.add(room.room_id) + logger.info("Agent dispatched to %s", lk_room_name) + except Exception: + logger.exception("Dispatch failed for %s", room.room_id) + + async def cleanup(self): + await self.client.close() + if self.lkapi: + await self.lkapi.aclose() + + +async def main(): + bot = Bot() + try: + await bot.start() + finally: + await bot.cleanup() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + asyncio.run(main()) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..bcd62ec --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +services: + agent: + build: . + command: python agent.py start + env_file: .env + restart: unless-stopped + network_mode: host + + bot: + build: . + command: python bot.py + env_file: .env + restart: unless-stopped diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..782f725 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +livekit-agents>=1.4,<2.0 +livekit-plugins-openai>=1.4,<2.0 +livekit-plugins-elevenlabs>=1.0,<2.0 +livekit-plugins-silero>=0.25,<1.0 +livekit>=0.18,<1.0 +livekit-api>=0.8,<1.0 +matrix-nio>=0.25,<1.0