feat(MAT-174): LLM-based result filtering for cron search jobs
Brave Search results are passed through LiteLLM (claude-haiku) when job config includes a `criteria` field. LLM returns indices of matching results, filtering out noise before posting to Matrix. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -148,3 +148,120 @@ class TestBraveSearchExecutor:
|
||||
bs.BRAVE_API_KEY = original_key
|
||||
|
||||
assert result["status"] == "no_results"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_filter_keeps_matching_results(self):
|
||||
"""LLM filter should only keep results that match criteria."""
|
||||
import cron.brave_search as bs
|
||||
orig_key, orig_url, orig_llm_key = bs.BRAVE_API_KEY, bs.LITELLM_URL, bs.LITELLM_KEY
|
||||
bs.BRAVE_API_KEY = "test-key"
|
||||
bs.LITELLM_URL = "http://llm:4000/v1"
|
||||
bs.LITELLM_KEY = "sk-test"
|
||||
|
||||
job = {
|
||||
"id": "j1",
|
||||
"name": "BMW Search",
|
||||
"jobType": "brave_search",
|
||||
"config": {"query": "BMW X3 damaged", "maxResults": 5, "criteria": "Must be BMW X3, petrol, <=2019, damaged"},
|
||||
"targetRoom": "!room:cars",
|
||||
"dedupKeys": [],
|
||||
}
|
||||
|
||||
brave_resp = MagicMock()
|
||||
brave_resp.json.return_value = {"web": {"results": [
|
||||
{"title": "BMW X3 2018 Unfallwagen Benzin", "url": "https://a.com", "description": "Damaged"},
|
||||
{"title": "Toyota Corolla 2020", "url": "https://b.com", "description": "Not a BMW"},
|
||||
{"title": "BMW X3 2017 Diesel crash", "url": "https://c.com", "description": "Diesel"},
|
||||
]}}
|
||||
brave_resp.raise_for_status = MagicMock()
|
||||
|
||||
llm_resp = MagicMock()
|
||||
llm_resp.json.return_value = {"choices": [{"message": {"content": "[0]"}}]}
|
||||
llm_resp.raise_for_status = MagicMock()
|
||||
|
||||
send_text = AsyncMock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_cls.return_value.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(return_value=brave_resp)
|
||||
mock_client.post = AsyncMock(return_value=llm_resp)
|
||||
|
||||
try:
|
||||
result = await execute_brave_search(job=job, send_text=send_text)
|
||||
finally:
|
||||
bs.BRAVE_API_KEY, bs.LITELLM_URL, bs.LITELLM_KEY = orig_key, orig_url, orig_llm_key
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["newDedupKeys"] == ["https://a.com"]
|
||||
msg = send_text.call_args[0][1]
|
||||
assert "Unfallwagen" in msg
|
||||
assert "Toyota" not in msg
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_filter_no_matches_returns_no_results(self):
|
||||
"""When LLM filter rejects all results, status should be no_results."""
|
||||
import cron.brave_search as bs
|
||||
orig_key, orig_url, orig_llm_key = bs.BRAVE_API_KEY, bs.LITELLM_URL, bs.LITELLM_KEY
|
||||
bs.BRAVE_API_KEY = "test-key"
|
||||
bs.LITELLM_URL = "http://llm:4000/v1"
|
||||
bs.LITELLM_KEY = "sk-test"
|
||||
|
||||
job = {
|
||||
"id": "j1", "name": "Search", "jobType": "brave_search",
|
||||
"config": {"query": "test", "criteria": "Must be exactly X"},
|
||||
"targetRoom": "!room:test", "dedupKeys": [],
|
||||
}
|
||||
|
||||
brave_resp = MagicMock()
|
||||
brave_resp.json.return_value = {"web": {"results": [{"title": "Nope", "url": "https://x.com", "description": "No"}]}}
|
||||
brave_resp.raise_for_status = MagicMock()
|
||||
|
||||
llm_resp = MagicMock()
|
||||
llm_resp.json.return_value = {"choices": [{"message": {"content": "[]"}}]}
|
||||
llm_resp.raise_for_status = MagicMock()
|
||||
|
||||
send_text = AsyncMock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_cls.return_value.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(return_value=brave_resp)
|
||||
mock_client.post = AsyncMock(return_value=llm_resp)
|
||||
|
||||
try:
|
||||
result = await execute_brave_search(job=job, send_text=send_text)
|
||||
finally:
|
||||
bs.BRAVE_API_KEY, bs.LITELLM_URL, bs.LITELLM_KEY = orig_key, orig_url, orig_llm_key
|
||||
|
||||
assert result["status"] == "no_results"
|
||||
send_text.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_criteria_skips_llm_filter(self, job):
|
||||
"""Without criteria, results pass through without LLM call."""
|
||||
import cron.brave_search as bs
|
||||
orig_key = bs.BRAVE_API_KEY
|
||||
bs.BRAVE_API_KEY = "test-key"
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = {"web": {"results": [{"title": "R", "url": "https://new.com", "description": "D"}]}}
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
send_text = AsyncMock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_cls.return_value.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
|
||||
try:
|
||||
result = await execute_brave_search(job=job, send_text=send_text)
|
||||
finally:
|
||||
bs.BRAVE_API_KEY = orig_key
|
||||
|
||||
assert result["status"] == "success"
|
||||
mock_client.post.assert_not_called()
|
||||
|
||||
Reference in New Issue
Block a user