fix: reduce sr.ht poll API load with cached repo discovery

2 files changed, 70 insertions, 11 deletions

diff --git a/src/srht_contrib/services/git.py b/src/srht_contrib/services/git.py
index 7d5360b..26b2957 100644
--- a/src/srht_contrib/services/git.py
+++ b/src/srht_contrib/services/git.py
@@ -6,7 +6,11 @@ from datetime import UTC, datetime, timedelta
 import logging
 from typing import Any
 
+from sqlalchemy import delete, select
+from sqlalchemy.orm import Session
+
 from srht_contrib.config import Settings
+from srht_contrib.models import DiscoveredRepository, TrackedRepository
 from srht_contrib.schemas import NormalizedEvent
 from srht_contrib.services.srht_client import SourceHutClientError, SourceHutGraphQLClient
 from srht_contrib.services.types import BackfillBatchResult
@@ -88,9 +92,10 @@ class GitIngestionService:
         actor: str,
         since: datetime | None = None,
         repositories: list[str] | None = None,
+        db: Session | None = None,
     ) -> GitPollResult:
         since_dt = ensure_utc(since or (datetime.now(tz=UTC) - timedelta(days=30)))
-        discovered_repositories = repositories or self._repositories_for_actor(actor)
+        discovered_repositories = repositories or self._repositories_for_actor(actor, db)
         if not discovered_repositories:
             logger.info("git poll skipped for actor=%s because no repositories were discovered", actor)
             return GitPollResult(events=[], cursor=datetime.now(tz=UTC).isoformat())
@@ -108,18 +113,44 @@ class GitIngestionService:
         logger.info("git poll complete for actor=%s normalized_events=%s", actor, len(events))
         return GitPollResult(events=events, cursor=datetime.now(tz=UTC).isoformat())
 
-    def _repositories_for_actor(self, actor: str) -> list[str]:
+    def _repositories_for_actor(self, actor: str, db: Session | None = None) -> list[str]:
         configured = {
             self._canonical_repository_name(actor, repository)
             for repository in self.settings.git_tracked_repositories
         }
-        discovered = set(self._discover_owned_repositories(actor))
-        repositories = sorted(configured | discovered)
+        tracked = set()
+        discovered = set()
+        now = datetime.now(tz=UTC)
+
+        if db is not None:
+            tracked = set(
+                db.scalars(
+                    select(TrackedRepository.repo_name)
+                    .where(TrackedRepository.service == self.service_name)
+                    .where(TrackedRepository.actor == actor)
+                ).all()
+            )
+            cutoff = now - timedelta(seconds=self.settings.git_repo_discovery_ttl_seconds)
+            discovered = set(
+                db.scalars(
+                    select(DiscoveredRepository.name)
+                    .where(DiscoveredRepository.actor == actor)
+                    .where(DiscoveredRepository.discovered_at > cutoff)
+                ).all()
+            )
+
+        if not discovered:
+            discovered = set(self._discover_owned_repositories(actor))
+            if db is not None:
+                self._refresh_discovered_repositories(db, actor, discovered, discovered_at=now)
+
+        repositories = sorted(configured | tracked | discovered)
         logger.info(
-            "git repositories selected for actor=%s count=%s configured=%s discovered=%s",
+            "git repositories selected for actor=%s count=%s configured=%s tracked=%s discovered=%s",
             actor,
             len(repositories),
             len(configured),
+            len(tracked),
             len(discovered),
         )
         return repositories
@@ -268,7 +299,8 @@ class GitIngestionService:
                 repository_owner = ((repository.get("owner") or {}).get("canonicalName") or actor).strip()
                 if not name or not repository_owner:
                     continue
-                repositories.append(f"{repository_owner}/{name}")
+                repo_name = f"{repository_owner}/{name}"
+                repositories.append(repo_name)
 
             if not cursor:
                 break
@@ -276,6 +308,25 @@ class GitIngestionService:
         return repositories
 
     @staticmethod
+    def _refresh_discovered_repositories(
+        db: Session,
+        actor: str,
+        repositories: set[str],
+        *,
+        discovered_at: datetime,
+    ) -> None:
+        db.execute(delete(DiscoveredRepository).where(DiscoveredRepository.actor == actor))
+        for repository in sorted(repositories):
+            db.add(
+                DiscoveredRepository(
+                    actor=actor,
+                    name=repository,
+                    discovered_at=discovered_at,
+                )
+            )
+        db.flush()
+
+    @staticmethod
     def _canonical_repository_name(default_actor: str, repository: str) -> str:
         owner, repo_name = GitIngestionService._split_repository(default_actor, repository)
         return f"~{owner}/{repo_name}"
diff --git a/src/srht_contrib/services/srht_client.py b/src/srht_contrib/services/srht_client.py
index ee09bff..1118f0d 100644
--- a/src/srht_contrib/services/srht_client.py
+++ b/src/srht_contrib/services/srht_client.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import time
 from typing import Any
 
 import httpx
@@ -27,11 +28,13 @@ class SourceHutGraphQLClient:
         *,
         timeout: float = 15.0,
         max_retries: int = 2,
+        request_delay: float = 0.5,
         transport: httpx.BaseTransport | None = None,
     ) -> None:
         self.endpoint = endpoint
         self.timeout = timeout
         self.max_retries = max_retries
+        self.request_delay = request_delay
         headers = {
             "Authorization": f"Bearer {token}",
             "Content-Type": "application/json",
@@ -42,7 +45,12 @@ class SourceHutGraphQLClient:
         payload = {"query": query, "variables": variables or {}}
         attempts = self.max_retries + 1
 
-        for attempt in range(1, attempts + 1):
+        for attempt in range(attempts):
+            if attempt > 0:
+                time.sleep(2 ** (attempt - 1))
+            elif self.request_delay > 0:
+                time.sleep(self.request_delay)
+
             try:
                 response = self._client.post(self.endpoint, json=payload)
                 response.raise_for_status()
@@ -51,21 +59,21 @@ class SourceHutGraphQLClient:
                 logger.warning(
                     "SourceHut HTTP failure from %s on attempt %s/%s: status=%s",
                     self.endpoint,
-                    attempt,
+                    attempt + 1,
                     attempts,
                     exc.response.status_code,
                 )
-                if exc.response.status_code >= 500 and attempt < attempts:
+                if exc.response.status_code >= 500 and attempt < attempts - 1:
                     continue
                 raise SourceHutClientError(f"HTTP error from SourceHut: {exc.response.status_code}") from exc
             except httpx.HTTPError as exc:
                 logger.warning(
                     "SourceHut network failure from %s on attempt %s/%s",
                     self.endpoint,
-                    attempt,
+                    attempt + 1,
                     attempts,
                 )
-                if attempt < attempts:
+                if attempt < attempts - 1:
                     continue
                 raise SourceHutClientError("Network error while contacting SourceHut") from exc