diff --git a/voice.py b/voice.py
index f38bdbd..c80b826 100644
--- a/voice.py
+++ b/voice.py
@@ -218,9 +218,12 @@ class VoiceSession:
             if self._publish_key_cb:
                 self._publish_key_cb(self._bot_key)
 
-            # Build E2EE options with empty shared key — we set per-participant
-            # keys after connect via e2ee_manager.key_provider.set_key()
-            e2ee_opts = _build_e2ee_options(b"")
+            # Connect with caller's key as shared_key for immediate decryption,
+            # then set per-participant keys after connect for proper separation
+            connect_key = self._caller_key or self._bot_key
+            e2ee_opts = _build_e2ee_options(connect_key)
+            logger.info("E2EE connect key: %d bytes (from %s)",
+                        len(connect_key), "caller" if self._caller_key else "bot")
 
             room_opts = rtc.RoomOptions(e2ee=e2ee_opts)
             self.lk_room = rtc.Room()
@@ -238,10 +241,10 @@ class VoiceSession:
                 logger.info("Track sub: %s %s kind=%s", p.identity, pub.sid, t.kind)
 
             await self.lk_room.connect(self.lk_url, jwt, options=room_opts)
-            logger.info("Connected (E2EE=per-participant), remote=%d",
+            logger.info("Connected (E2EE=shared+per-participant), remote=%d",
                         len(self.lk_room.remote_participants))
 
-            # Set per-participant E2EE keys via key provider
+            # Set per-participant E2EE keys after connect
             bot_identity = _make_lk_identity(user_id, self.device_id)
             try:
                 kp = self.lk_room.e2ee_manager.key_provider
@@ -257,26 +260,14 @@ class VoiceSession:
                     logger.info("Set caller E2EE key for identity=%s (%d bytes)",
                                 self._caller_identity, len(self._caller_key))
                 elif self._caller_key:
-                    # Caller identity not yet known — try to get from remote participants
                     for p in self.lk_room.remote_participants.values():
                         kp.set_key(p.identity, self._caller_key, key_index=0)
                         logger.info("Set caller E2EE key for identity=%s (%d bytes)",
                                     p.identity, len(self._caller_key))
                         break
-                else:
-                    logger.warning("No caller E2EE key available — caller audio will be silent")
-            except AttributeError:
-                logger.warning("e2ee_manager.key_provider not available — "
-                               "falling back to shared key mode")
-                # Fallback: set shared key after connect if per-participant isn't supported
-                if self._caller_key:
-                    try:
-                        kp = self.lk_room.e2ee_manager.key_provider
-                        kp.set_shared_key(self._caller_key, key_index=0)
-                        logger.info("Fallback: set shared E2EE key (%d bytes)",
-                                    len(self._caller_key))
-                    except Exception:
-                        logger.exception("Fallback shared key also failed")
+            except Exception:
+                logger.warning("Per-participant key setup failed, shared key used as fallback",
+                               exc_info=True)
 
             # Find the remote participant, wait up to 10s if not yet connected
             remote_identity = None