feat(MAT-174): LLM-based result filtering for cron search jobs

Brave Search results are passed through LiteLLM (claude-haiku) when
job config includes a `criteria` field. LLM returns indices of matching
results, filtering out noise before posting to Matrix.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Christian Gick
2026-03-16 10:14:01 +02:00
parent 846634738b
commit 19b72dfe07
2 changed files with 191 additions and 2 deletions

View File

@@ -148,3 +148,120 @@ class TestBraveSearchExecutor:
bs.BRAVE_API_KEY = original_key
assert result["status"] == "no_results"
@pytest.mark.asyncio
async def test_llm_filter_keeps_matching_results(self):
"""LLM filter should only keep results that match criteria."""
import cron.brave_search as bs
orig_key, orig_url, orig_llm_key = bs.BRAVE_API_KEY, bs.LITELLM_URL, bs.LITELLM_KEY
bs.BRAVE_API_KEY = "test-key"
bs.LITELLM_URL = "http://llm:4000/v1"
bs.LITELLM_KEY = "sk-test"
job = {
"id": "j1",
"name": "BMW Search",
"jobType": "brave_search",
"config": {"query": "BMW X3 damaged", "maxResults": 5, "criteria": "Must be BMW X3, petrol, <=2019, damaged"},
"targetRoom": "!room:cars",
"dedupKeys": [],
}
brave_resp = MagicMock()
brave_resp.json.return_value = {"web": {"results": [
{"title": "BMW X3 2018 Unfallwagen Benzin", "url": "https://a.com", "description": "Damaged"},
{"title": "Toyota Corolla 2020", "url": "https://b.com", "description": "Not a BMW"},
{"title": "BMW X3 2017 Diesel crash", "url": "https://c.com", "description": "Diesel"},
]}}
brave_resp.raise_for_status = MagicMock()
llm_resp = MagicMock()
llm_resp.json.return_value = {"choices": [{"message": {"content": "[0]"}}]}
llm_resp.raise_for_status = MagicMock()
send_text = AsyncMock()
with patch("httpx.AsyncClient") as mock_cls:
mock_client = AsyncMock()
mock_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_cls.return_value.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=brave_resp)
mock_client.post = AsyncMock(return_value=llm_resp)
try:
result = await execute_brave_search(job=job, send_text=send_text)
finally:
bs.BRAVE_API_KEY, bs.LITELLM_URL, bs.LITELLM_KEY = orig_key, orig_url, orig_llm_key
assert result["status"] == "success"
assert result["newDedupKeys"] == ["https://a.com"]
msg = send_text.call_args[0][1]
assert "Unfallwagen" in msg
assert "Toyota" not in msg
@pytest.mark.asyncio
async def test_llm_filter_no_matches_returns_no_results(self):
"""When LLM filter rejects all results, status should be no_results."""
import cron.brave_search as bs
orig_key, orig_url, orig_llm_key = bs.BRAVE_API_KEY, bs.LITELLM_URL, bs.LITELLM_KEY
bs.BRAVE_API_KEY = "test-key"
bs.LITELLM_URL = "http://llm:4000/v1"
bs.LITELLM_KEY = "sk-test"
job = {
"id": "j1", "name": "Search", "jobType": "brave_search",
"config": {"query": "test", "criteria": "Must be exactly X"},
"targetRoom": "!room:test", "dedupKeys": [],
}
brave_resp = MagicMock()
brave_resp.json.return_value = {"web": {"results": [{"title": "Nope", "url": "https://x.com", "description": "No"}]}}
brave_resp.raise_for_status = MagicMock()
llm_resp = MagicMock()
llm_resp.json.return_value = {"choices": [{"message": {"content": "[]"}}]}
llm_resp.raise_for_status = MagicMock()
send_text = AsyncMock()
with patch("httpx.AsyncClient") as mock_cls:
mock_client = AsyncMock()
mock_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_cls.return_value.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=brave_resp)
mock_client.post = AsyncMock(return_value=llm_resp)
try:
result = await execute_brave_search(job=job, send_text=send_text)
finally:
bs.BRAVE_API_KEY, bs.LITELLM_URL, bs.LITELLM_KEY = orig_key, orig_url, orig_llm_key
assert result["status"] == "no_results"
send_text.assert_not_called()
@pytest.mark.asyncio
async def test_no_criteria_skips_llm_filter(self, job):
"""Without criteria, results pass through without LLM call."""
import cron.brave_search as bs
orig_key = bs.BRAVE_API_KEY
bs.BRAVE_API_KEY = "test-key"
mock_response = MagicMock()
mock_response.json.return_value = {"web": {"results": [{"title": "R", "url": "https://new.com", "description": "D"}]}}
mock_response.raise_for_status = MagicMock()
send_text = AsyncMock()
with patch("httpx.AsyncClient") as mock_cls:
mock_client = AsyncMock()
mock_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_cls.return_value.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=mock_response)
try:
result = await execute_brave_search(job=job, send_text=send_text)
finally:
bs.BRAVE_API_KEY = orig_key
assert result["status"] == "success"
mock_client.post.assert_not_called()