diff options
| -rw-r--r-- | API.md | 1 | ||||
| -rw-r--r-- | README.md | 16 | ||||
| -rw-r--r-- | src/srht_contrib/services/git.py | 83 | ||||
| -rw-r--r-- | tests/test_ingestion.py | 61 |
4 files changed, 146 insertions, 15 deletions
@@ -51,6 +51,7 @@ Repository names: - shorthand `repo-name` - canonical `~owner/repo-name` - Stored repository names are normalized to canonical `~owner/repo-name` form. +- Tracked repositories are optional force-includes for git polling; owned repositories are auto-discovered per actor. ## Health @@ -8,7 +8,7 @@ The current V1 is intentionally narrow and production-oriented: - SQLite-backed persistence - polling-based ingestion - complete `todo.sr.ht` ingestion path -- practical `git.sr.ht` commit ingestion for tracked repositories +- practical `git.sr.ht` commit ingestion with automatic repository discovery - public read-only contribution endpoints plus API-key protection for mutating/admin routes - Alembic-managed schema migrations @@ -51,7 +51,7 @@ Current normalized event types: - `ticket_closed` - `commit` -`todo.sr.ht` uses a feed-first strategy and falls back to crawling the authenticated user’s trackers, tickets, and ticket events when the top-level activity feed is empty. `git.sr.ht` polls tracked repositories for recent commits on the default branch. +`todo.sr.ht` uses a feed-first strategy and falls back to crawling the authenticated user’s trackers, tickets, and ticket events when the top-level activity feed is empty. `git.sr.ht` polls the actor's owned repositories for recent commits on the default branch and unions in any explicitly configured repositories. ## Canonical Event Model @@ -69,7 +69,7 @@ All ingestion services normalize external activity into this shape: The database enforces uniqueness on `(service, external_uid)` so polling is safe to repeat. -Tracked git repositories are persisted in the `tracked_repositories` table and stored in canonical `~owner/repo` form. The poller seeds that table from `GIT_TRACKED_REPOSITORIES`, and repositories can also be created, updated, and deleted through the API. +Tracked git repositories are persisted in the `tracked_repositories` table and stored in canonical `~owner/repo` form. They are optional overrides now: the poller auto-discovers an actor's owned repositories and unions in any configured or API-managed repositories. ## Configuration @@ -84,7 +84,7 @@ Environment variables: - `DEFAULT_ACTOR`: actor used by the scheduled poll job - `POLL_INTERVAL_SECONDS`: scheduler interval in seconds - `ACTOR_ALIASES_JSON`: optional JSON object for actor/email/display-name alias mapping -- `GIT_TRACKED_REPOSITORIES`: optional JSON array of repository names or `owner/repo` strings for git polling +- `GIT_TRACKED_REPOSITORIES`: optional JSON array of repository names or `owner/repo` strings to union into git polling Example `.env`: @@ -132,7 +132,7 @@ Set at least: - `API_KEY` - `SRHT_TOKEN` - `DEFAULT_ACTOR` -- `GIT_TRACKED_REPOSITORIES` if you want git commit ingestion +- `GIT_TRACKED_REPOSITORIES` if you want to force-include extra repositories beyond the actor's owned repos ### 3. Run database migrations @@ -167,7 +167,7 @@ Example response: Scheduled polling only runs when `ENABLE_SCHEDULER=true`. The scheduler seeds `DEFAULT_ACTOR` as an initial known actor, and public contribution reads register additional actors for later background polling. -For `git.sr.ht`, tracked repositories are configured via `GIT_TRACKED_REPOSITORIES`. Entries may be either: +For `git.sr.ht`, owned repositories are auto-discovered for the actor. `GIT_TRACKED_REPOSITORIES` can still be used to union in extra repositories. Entries may be either: - `"Hutch"` for a repository owned by `DEFAULT_ACTOR` - `"~your-user/your-site"` for an explicit owner/repository pair @@ -310,11 +310,11 @@ Covered areas: The SourceHut-specific assumptions are isolated to the service modules: - `src/srht_contrib/services/todo.py` uses the authenticated `events(cursor)` feed first, then falls back to tracker/ticket event traversal for reliable contribution discovery. -- `src/srht_contrib/services/git.py` uses the documented repository `log(cursor)` query against tracked repositories and attributes commits through the configured alias map. +- `src/srht_contrib/services/git.py` discovers owned repositories for an actor, polls each repository `log(cursor)`, and attributes commits through the configured alias map. ## Known Limitations -- `git.sr.ht` polling is limited to repositories listed in `GIT_TRACKED_REPOSITORIES` +- `git.sr.ht` polling assumes the actor's repositories are discoverable through the SourceHut GraphQL API - scheduled polling runs in-process, so it is not a distributed scheduler - newly requested actors are indexed asynchronously, so the first public read may be empty until a scheduler or manual poll runs - alias management is config-driven; there is no alias CRUD API yet diff --git a/src/srht_contrib/services/git.py b/src/srht_contrib/services/git.py index 20a18fe..9531709 100644 --- a/src/srht_contrib/services/git.py +++ b/src/srht_contrib/services/git.py @@ -47,6 +47,23 @@ query RepositoryLog($username: String!, $repoName: String!, $cursor: Cursor) { """.strip() +USER_REPOSITORIES_QUERY = """ +query UserRepositories($username: String!, $cursor: Cursor) { + user(username: $username) { + repositories(cursor: $cursor) { + results { + name + owner { + canonicalName + } + } + cursor + } + } +} +""".strip() + + @dataclass(slots=True) class GitPollResult: events: list[NormalizedEvent] @@ -54,7 +71,7 @@ class GitPollResult: class GitIngestionService: - """Polls tracked git.sr.ht repositories and normalizes commits for one actor.""" + """Polls git.sr.ht repositories and normalizes commits for one actor.""" service_name = "git" @@ -70,13 +87,13 @@ class GitIngestionService: repositories: list[str] | None = None, ) -> GitPollResult: since_dt = ensure_utc(since or (datetime.now(tz=UTC) - timedelta(days=30))) - tracked_repositories = repositories or self._tracked_repositories(actor) - if not tracked_repositories: - logger.info("git poll skipped for actor=%s because no tracked repositories are configured", actor) + discovered_repositories = repositories or self._repositories_for_actor(actor) + if not discovered_repositories: + logger.info("git poll skipped for actor=%s because no repositories were discovered", actor) return GitPollResult(events=[], cursor=datetime.now(tz=UTC).isoformat()) events: list[NormalizedEvent] = [] - for repository in tracked_repositories: + for repository in discovered_repositories: owner, repo_name = self._split_repository(actor, repository) repo_events = self._fetch_repository_commits(actor=actor, owner=owner, repo_name=repo_name, since=since_dt) events.extend(repo_events) @@ -84,10 +101,62 @@ class GitIngestionService: logger.info("git poll complete for actor=%s normalized_events=%s", actor, len(events)) return GitPollResult(events=events, cursor=datetime.now(tz=UTC).isoformat()) - def _tracked_repositories(self, actor: str) -> list[str]: - repositories = self.settings.git_tracked_repositories + def _repositories_for_actor(self, actor: str) -> list[str]: + configured = { + self._canonical_repository_name(actor, repository) + for repository in self.settings.git_tracked_repositories + } + discovered = set(self._discover_owned_repositories(actor)) + repositories = sorted(configured | discovered) + logger.info( + "git repositories selected for actor=%s count=%s configured=%s discovered=%s", + actor, + len(repositories), + len(configured), + len(discovered), + ) return repositories + def _discover_owned_repositories(self, actor: str) -> list[str]: + owner = actor.lstrip("~") + repositories: list[str] = [] + cursor: str | None = None + + for _ in range(50): + data = self.client.execute( + USER_REPOSITORIES_QUERY, + {"username": owner, "cursor": cursor}, + ) + user = data.get("user") or {} + repositories_page = user.get("repositories") or {} + results = repositories_page.get("results") or [] + cursor = repositories_page.get("cursor") + logger.info( + "git repository discovery actor=%s page_count=%s next_cursor=%s", + actor, + len(results), + bool(cursor), + ) + + for repository in results: + if not isinstance(repository, dict): + continue + name = repository.get("name") + repository_owner = ((repository.get("owner") or {}).get("canonicalName") or actor).strip() + if not name or not repository_owner: + continue + repositories.append(f"{repository_owner}/{name}") + + if not cursor: + break + + return repositories + + @staticmethod + def _canonical_repository_name(default_actor: str, repository: str) -> str: + owner, repo_name = GitIngestionService._split_repository(default_actor, repository) + return f"~{owner}/{repo_name}" + @staticmethod def _split_repository(default_actor: str, repository: str) -> tuple[str, str]: if "/" in repository: diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py index 788ec81..3b7db16 100644 --- a/tests/test_ingestion.py +++ b/tests/test_ingestion.py @@ -310,6 +310,67 @@ def test_git_ingestion_normalizes_commit_aliases_and_repository_names(db_session assert tracked_repositories == ["~ccleberg/Hutch"] +def test_git_ingestion_auto_discovers_owned_repositories(db_session) -> None: + settings = make_settings( + ACTOR_ALIASES_JSON={"~ccleberg": ["[email protected]", "Chris Cleberg"]}, + GIT_TRACKED_REPOSITORIES=[], + ) + client = StubClient( + payloads_by_query={ + "query UserRepositories": { + "user": { + "repositories": { + "results": [ + {"name": "Hutch", "owner": {"canonicalName": "~ccleberg"}}, + ], + "cursor": None, + } + } + }, + "query RepositoryLog": { + "user": { + "repository": { + "name": "Hutch", + "owner": {"canonicalName": "~ccleberg"}, + "log": { + "results": [ + { + "id": "abc123", + "shortId": "abc123", + "author": { + "name": "Chris Cleberg", + "email": "[email protected]", + "time": "2026-03-30T12:00:00Z", + }, + "committer": { + "name": "Chris Cleberg", + "email": "[email protected]", + "time": "2026-03-30T12:00:00Z", + }, + "message": "Auto-discovered repo commit", + } + ], + "cursor": None, + }, + } + } + }, + } + ) + + todo_service = TodoIngestionService( + StubClient(payload={"me": {"canonicalName": "~ccleberg"}, "events": {"results": [], "cursor": None}}), + settings, + ) + git_service = GitIngestionService(client, settings) + poller = PollerService(todo_service=todo_service, git_service=git_service) + + inserted = poller.poll_all(db_session, "~ccleberg") + + assert inserted == 1 + assert any("query UserRepositories" in call[0] for call in client.calls) + + def test_sync_overlap_reuses_cursor_window_and_suppresses_duplicates(db_session) -> None: event = NormalizedEvent( service="todo", |