diff --git a/voice.py b/voice.py
index c80b826..05c7298 100644
--- a/voice.py
+++ b/voice.py
@@ -107,11 +107,7 @@ class VoiceSession:
         self._publish_key_cb = publish_key_cb
 
     def on_encryption_key(self, sender, device_id, key, index):
-        """Receive E2EE key from Element Call participant.
-
-        If the room is already connected, immediately set the key on the
-        key provider so we can decrypt the caller's audio.
-        """
+        """Receive E2EE key from Element Call participant."""
         if not key:
             return
         identity = _make_lk_identity(sender, device_id)
@@ -120,15 +116,6 @@ class VoiceSession:
         logger.info("E2EE key received from %s:%s (identity=%s, index=%d, %d bytes)",
                     sender, device_id, identity, index, len(key))
 
-        # If already connected, set key on the key provider immediately
-        if self.lk_room:
-            try:
-                kp = self.lk_room.e2ee_manager.key_provider
-                kp.set_key(identity, key, key_index=index)
-                logger.info("Live-updated caller E2EE key for %s", identity)
-            except Exception:
-                logger.warning("Could not live-update caller E2EE key", exc_info=True)
-
     async def _fetch_encryption_key_http(self) -> bytes | None:
         """Fetch encryption key from room timeline (NOT state) via Matrix HTTP API.
 
@@ -214,16 +201,16 @@ class VoiceSession:
                         break
                     await asyncio.sleep(0.1)
 
-            # Publish bot's own key so caller can decrypt our audio
+            # Use caller's key for both directions (shared key mode).
+            # Bot publishes the SAME key so caller can decrypt our audio too.
+            if self._caller_key:
+                self._bot_key = self._caller_key  # reuse caller's key
+                logger.info("Using caller's key as shared key for both directions (%d bytes)",
+                            len(self._caller_key))
             if self._publish_key_cb:
                 self._publish_key_cb(self._bot_key)
 
-            # Connect with caller's key as shared_key for immediate decryption,
-            # then set per-participant keys after connect for proper separation
-            connect_key = self._caller_key or self._bot_key
-            e2ee_opts = _build_e2ee_options(connect_key)
-            logger.info("E2EE connect key: %d bytes (from %s)",
-                        len(connect_key), "caller" if self._caller_key else "bot")
+            e2ee_opts = _build_e2ee_options(self._bot_key)
 
             room_opts = rtc.RoomOptions(e2ee=e2ee_opts)
             self.lk_room = rtc.Room()
@@ -241,34 +228,9 @@ class VoiceSession:
                 logger.info("Track sub: %s %s kind=%s", p.identity, pub.sid, t.kind)
 
             await self.lk_room.connect(self.lk_url, jwt, options=room_opts)
-            logger.info("Connected (E2EE=shared+per-participant), remote=%d",
+            logger.info("Connected (E2EE=shared key), remote=%d",
                         len(self.lk_room.remote_participants))
 
-            # Set per-participant E2EE keys after connect
-            bot_identity = _make_lk_identity(user_id, self.device_id)
-            try:
-                kp = self.lk_room.e2ee_manager.key_provider
-
-                # Set bot's own key (encrypts outgoing audio)
-                kp.set_key(bot_identity, self._bot_key, key_index=0)
-                logger.info("Set bot E2EE key for identity=%s (%d bytes)",
-                            bot_identity, len(self._bot_key))
-
-                # Set caller's key (decrypts incoming audio)
-                if self._caller_key and self._caller_identity:
-                    kp.set_key(self._caller_identity, self._caller_key, key_index=0)
-                    logger.info("Set caller E2EE key for identity=%s (%d bytes)",
-                                self._caller_identity, len(self._caller_key))
-                elif self._caller_key:
-                    for p in self.lk_room.remote_participants.values():
-                        kp.set_key(p.identity, self._caller_key, key_index=0)
-                        logger.info("Set caller E2EE key for identity=%s (%d bytes)",
-                                    p.identity, len(self._caller_key))
-                        break
-            except Exception:
-                logger.warning("Per-participant key setup failed, shared key used as fallback",
-                               exc_info=True)
-
             # Find the remote participant, wait up to 10s if not yet connected
             remote_identity = None
             for p in self.lk_room.remote_participants.values():