fix: reduce sr.ht poll API load with cached repo discovery

4 files changed, 160 insertions, 11 deletions

diff --git a/tests/test_git_repository_cache.py b/tests/test_git_repository_cache.py
new file mode 100644
index 0000000..98a3db3
--- /dev/null
+++ b/tests/test_git_repository_cache.py
@@ -0,0 +1,107 @@
+from datetime import UTC, datetime
+
+from sqlalchemy import select
+
+from srht_contrib.config import Settings
+from srht_contrib.jobs.poller import PollerService
+from srht_contrib.models import DiscoveredRepository
+from srht_contrib.services.git import GitIngestionService
+from srht_contrib.services.todo import TodoIngestionService
+
+
+class StubClient:
+    def __init__(self, payload: dict | None = None, payloads_by_query: dict[str, dict] | None = None) -> None:
+        self.payload = payload or {}
+        self.payloads_by_query = payloads_by_query or {}
+        self.calls: list[tuple[str, dict | None]] = []
+
+    def execute(self, query: str, variables: dict | None = None) -> dict:
+        self.calls.append((query, variables))
+        for marker, payload in self.payloads_by_query.items():
+            if marker in query:
+                return payload
+        return self.payload
+
+
+def make_settings(**overrides) -> Settings:
+    values = {
+        "API_KEY": "test-api-key",
+        "ENABLE_SCHEDULER": False,
+        "SRHT_TOKEN": "x",
+        "DATABASE_URL": "sqlite://",
+        "DEFAULT_ACTOR": "~ccleberg",
+        "TODO_SRHT_ENDPOINT": "https://todo.sr.ht/query",
+        "GIT_SRHT_ENDPOINT": "https://git.sr.ht/query",
+        "POLL_INTERVAL_SECONDS": 60,
+        "GIT_TRACKED_REPOSITORIES": [],
+        "ACTOR_ALIASES_JSON": {"~ccleberg": ["[email protected]", "Chris Cleberg"]},
+    }
+    values.update(overrides)
+    return Settings(**values)
+
+
+def test_git_poll_reuses_cached_discovered_repositories(db_session) -> None:
+    settings = make_settings()
+    client = StubClient(
+        payloads_by_query={
+            "query UserRepositories": {
+                "user": {
+                    "repositories": {
+                        "results": [
+                            {
+                                "name": "Hutch",
+                                "visibility": "PUBLIC",
+                                "owner": {"canonicalName": "~ccleberg"},
+                            }
+                        ],
+                        "cursor": None,
+                    }
+                }
+            },
+            "query RepositoryLog": {
+                "user": {
+                    "repository": {
+                        "name": "Hutch",
+                        "owner": {"canonicalName": "~ccleberg"},
+                        "log": {
+                            "results": [
+                                {
+                                    "id": "abc123",
+                                    "shortId": "abc123",
+                                    "author": {
+                                        "name": "Chris Cleberg",
+                                        "email": "[email protected]",
+                                        "time": "2026-03-30T12:00:00Z",
+                                    },
+                                    "committer": {
+                                        "name": "Chris Cleberg",
+                                        "email": "[email protected]",
+                                        "time": "2026-03-30T12:00:00Z",
+                                    },
+                                    "message": "Auto-discovered repo commit",
+                                }
+                            ],
+                            "cursor": None,
+                        },
+                    }
+                }
+            },
+        }
+    )
+    todo_service = TodoIngestionService(
+        StubClient(payload={"me": {"canonicalName": "~ccleberg"}, "events": {"results": [], "cursor": None}}),
+        settings,
+    )
+    git_service = GitIngestionService(client, settings)
+    poller = PollerService(todo_service=todo_service, git_service=git_service, settings=settings)
+
+    first_inserted = poller.poll_all(db_session, "~ccleberg")
+    second_inserted = poller.poll_all(db_session, "~ccleberg")
+
+    user_repository_calls = [call for call in client.calls if "query UserRepositories" in call[0]]
+    cached_names = db_session.scalars(select(DiscoveredRepository.name)).all()
+
+    assert first_inserted == 1
+    assert second_inserted == 0
+    assert len(user_repository_calls) == 1
+    assert cached_names == ["~ccleberg/Hutch"]
diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py
index aa79e3c..53848e7 100644
--- a/tests/test_ingestion.py
+++ b/tests/test_ingestion.py
@@ -274,7 +274,7 @@ def test_todo_ingestion_is_idempotent(db_session) -> None:
 
     todo_service = TodoIngestionService(StubClient(payload), settings)
     git_service = GitIngestionService(StubClient(payload={}), settings)
-    poller = PollerService(todo_service=todo_service, git_service=git_service)
+    poller = PollerService(todo_service=todo_service, git_service=git_service, settings=settings)
 
     first_inserted = poller.poll_all(db_session, "~ccleberg")
     second_inserted = poller.poll_all(db_session, "~ccleberg")
@@ -348,7 +348,7 @@ def test_todo_ingestion_falls_back_to_tracker_crawl(db_session) -> None:
     )
     todo_service = TodoIngestionService(client, settings)
     git_service = GitIngestionService(StubClient(payload={}), settings)
-    poller = PollerService(todo_service=todo_service, git_service=git_service)
+    poller = PollerService(todo_service=todo_service, git_service=git_service, settings=settings)
 
     inserted = poller.poll_all(db_session, "~ccleberg")
 
@@ -384,7 +384,7 @@ def test_unsupported_todo_changes_are_ignored(db_session) -> None:
 
     todo_service = TodoIngestionService(StubClient(payload), settings)
     git_service = GitIngestionService(StubClient(payload={}), settings)
-    poller = PollerService(todo_service=todo_service, git_service=git_service)
+    poller = PollerService(todo_service=todo_service, git_service=git_service, settings=settings)
 
     inserted = poller.poll_all(db_session, "~ccleberg")
 
@@ -430,7 +430,7 @@ def test_git_ingestion_normalizes_commit_aliases_and_repository_names(db_session
         settings,
     )
     git_service = GitIngestionService(StubClient(payloads_by_query={"query RepositoryLog": git_payload}), settings)
-    poller = PollerService(todo_service=todo_service, git_service=git_service)
+    poller = PollerService(todo_service=todo_service, git_service=git_service, settings=settings)
 
     inserted = poller.poll_all(db_session, "~ccleberg")
 
@@ -451,7 +451,11 @@ def test_git_ingestion_auto_discovers_owned_repositories(db_session) -> None:
                 "user": {
                     "repositories": {
                         "results": [
-                            {"name": "Hutch", "owner": {"canonicalName": "~ccleberg"}},
+                            {
+                                "name": "Hutch",
+                                "visibility": "PUBLIC",
+                                "owner": {"canonicalName": "~ccleberg"},
+                            },
                         ],
                         "cursor": None,
                     }
@@ -493,7 +497,7 @@ def test_git_ingestion_auto_discovers_owned_repositories(db_session) -> None:
         settings,
     )
     git_service = GitIngestionService(client, settings)
-    poller = PollerService(todo_service=todo_service, git_service=git_service)
+    poller = PollerService(todo_service=todo_service, git_service=git_service, settings=settings)
 
     inserted = poller.poll_all(db_session, "~ccleberg")
 
@@ -502,6 +506,7 @@ def test_git_ingestion_auto_discovers_owned_repositories(db_session) -> None:
 
 
 def test_sync_overlap_reuses_cursor_window_and_suppresses_duplicates(db_session) -> None:
+    settings = make_settings()
     event = NormalizedEvent(
         service="todo",
         event_type="ticket_created",
@@ -514,7 +519,7 @@ def test_sync_overlap_reuses_cursor_window_and_suppresses_duplicates(db_session)
         raw_payload_json=None,
     )
     todo_service = RecordingTodoService(events_by_call=[[event], [event]])
-    poller = PollerService(todo_service=todo_service, git_service=EmptyGitService())
+    poller = PollerService(todo_service=todo_service, git_service=EmptyGitService(), settings=settings)
 
     first_inserted = poller.poll_all(db_session, "~ccleberg")
     second_inserted = poller.poll_all(db_session, "~ccleberg")
@@ -529,6 +534,7 @@ def test_sync_overlap_reuses_cursor_window_and_suppresses_duplicates(db_session)
 
 
 def test_scheduled_poll_polls_known_actors_and_seeds_default_actor(db_session) -> None:
+    settings = make_settings()
     event = NormalizedEvent(
         service="todo",
         event_type="ticket_created",
@@ -541,7 +547,7 @@ def test_scheduled_poll_polls_known_actors_and_seeds_default_actor(db_session) -
         raw_payload_json=None,
     )
     todo_service = RecordingTodoService(events_by_call=[[], [event]])
-    poller = PollerService(todo_service=todo_service, git_service=EmptyGitService())
+    poller = PollerService(todo_service=todo_service, git_service=EmptyGitService(), settings=settings)
 
     db_session.add(TrackedActor(actor="~known", is_active=True))
     db_session.commit()
@@ -557,7 +563,8 @@ def test_scheduled_poll_polls_known_actors_and_seeds_default_actor(db_session) -
 
 
 def test_poll_marks_backfill_complete_and_persists_service_state(db_session) -> None:
-    poller = PollerService(todo_service=BackfillingTodoService(), git_service=EmptyGitService())
+    settings = make_settings()
+    poller = PollerService(todo_service=BackfillingTodoService(), git_service=EmptyGitService(), settings=settings)
 
     inserted = poller.poll_all(db_session, "~ccleberg")
 
@@ -577,7 +584,12 @@ def test_poll_marks_backfill_complete_and_persists_service_state(db_session) ->
 
 
 def test_backfill_cursor_state_shrinks_across_repeated_polls(db_session) -> None:
-    poller = PollerService(todo_service=QueueShrinkingTodoService(), git_service=QueueShrinkingGitService())
+    settings = make_settings()
+    poller = PollerService(
+        todo_service=QueueShrinkingTodoService(),
+        git_service=QueueShrinkingGitService(),
+        settings=settings,
+    )
 
     poller.poll_all(db_session, "~ccleberg")
     first_states = {
@@ -606,7 +618,8 @@ def test_backfill_cursor_state_shrinks_across_repeated_polls(db_session) -> None
 
 
 def test_prune_old_events_removes_data_older_than_one_year(db_session) -> None:
-    poller = PollerService(todo_service=BackfillingTodoService(), git_service=EmptyGitService())
+    settings = make_settings()
+    poller = PollerService(todo_service=BackfillingTodoService(), git_service=EmptyGitService(), settings=settings)
     db_session.add_all(
         [
             ContributionEvent(
diff --git a/tests/test_migrations.py b/tests/test_migrations.py
index 6173693..a3091fc 100644
--- a/tests/test_migrations.py
+++ b/tests/test_migrations.py
@@ -21,6 +21,7 @@ def test_alembic_upgrade_creates_schema(tmp_path) -> None:
         "actor_aliases",
         "alembic_version",
         "contribution_events",
+        "discovered_repositories",
         "service_backfill_states",
         "sync_states",
         "tracked_actors",
@@ -117,6 +118,7 @@ def test_alembic_upgrade_adopts_legacy_schema(tmp_path) -> None:
     assert columns["actor"]["nullable"] is False
     assert "uq_tracked_repository_service_actor_name" in unique_constraints
     assert actor == Settings().default_actor
+    assert "discovered_repositories" in inspector.get_table_names()
     assert "tracked_actors" in inspector.get_table_names()
     assert "service_backfill_states" in inspector.get_table_names()
 
@@ -132,5 +134,6 @@ def test_alembic_prefers_database_url_from_environment(tmp_path, monkeypatch) ->
 
     inspector = inspect(create_engine(database_url))
     assert "actor_aliases" in inspector.get_table_names()
+    assert "discovered_repositories" in inspector.get_table_names()
     assert "tracked_actors" in inspector.get_table_names()
     assert "service_backfill_states" in inspector.get_table_names()
diff --git a/tests/test_srht_client.py b/tests/test_srht_client.py
index e7ece0d..b4586e5 100644
--- a/tests/test_srht_client.py
+++ b/tests/test_srht_client.py
@@ -54,3 +54,29 @@ def test_graphql_client_raises_for_network_errors() -> None:
         client.execute("query Ping { ping }")
 
     client.close()
+
+
+def test_graphql_client_applies_request_delay_and_retry_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
+    attempts = {"count": 0}
+    sleeps: list[float] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        attempts["count"] += 1
+        if attempts["count"] < 3:
+            return httpx.Response(502, json={"error": "bad gateway"})
+        return httpx.Response(200, json={"data": {"ok": True}})
+
+    monkeypatch.setattr("srht_contrib.services.srht_client.time.sleep", sleeps.append)
+    client = SourceHutGraphQLClient(
+        "https://todo.sr.ht/query",
+        "token",
+        max_retries=2,
+        request_delay=0.5,
+        transport=httpx.MockTransport(handler),
+    )
+
+    data = client.execute("query Ping { ping }")
+
+    assert data == {"ok": True}
+    assert sleeps == [0.5, 1, 2]
+    client.close()