From 5ef7200e92e4f0f7fc971ba4b7ef8d6d9173752e Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Mon, 24 Aug 2026 16:59:24 +0800 Subject: [PATCH] feat(github): synchronize Stars and Lists - replace the legacy PoC with canonical Account, Repository, and List graph facts - reconcile complete authenticated snapshots through batched Block and Relation persistence - ship GitHub extension 0.2.0 with real-account acceptance evidence --- .changes/github/0.2.0.md | 3 + app/business/info_base/block.py | 19 ++ app/business/info_base/relation.py | 17 +- extensions/github/CHANGELOG.md | 4 + extensions/github/README.md | 55 +++-- extensions/github/__init__.py | 25 +-- extensions/github/adapter.py | 324 +++++++++++++++++++++++++++ extensions/github/pyproject.toml | 4 +- extensions/github/repository.py | 344 +++++++++++++++++++++++++++++ extensions/github/resolver.py | 286 ++++++++++++++---------- extensions/github/schema.py | 160 +++++++++----- extensions/github/stars.py | 226 ++----------------- pdm.lock | 140 +----------- pyproject.toml | 1 - 14 files changed, 1028 insertions(+), 580 deletions(-) create mode 100644 .changes/github/0.2.0.md create mode 100644 extensions/github/adapter.py create mode 100644 extensions/github/repository.py diff --git a/.changes/github/0.2.0.md b/.changes/github/0.2.0.md new file mode 100644 index 0000000..f901315 --- /dev/null +++ b/.changes/github/0.2.0.md @@ -0,0 +1,3 @@ +## 0.2.0 - 2026-08-24 +### Added +* Synchronize the authenticated account's Stars, Lists, memberships, and repository ownership as reusable graph facts. diff --git a/app/business/info_base/block.py b/app/business/info_base/block.py index 692d0f4..d891b64 100644 --- a/app/business/info_base/block.py +++ b/app/business/info_base/block.py @@ -136,6 +136,25 @@ def create(cls, form: BlockForm, db_session: Opt[sqlmodel.Session] = None) -> Bl ) return block + @classmethod + def create_many( + cls, + forms: typing.Iterable[BlockForm], + db_session: sqlmodel.Session, + ) -> tuple[BlockModel, ...]: + """Create a caller-owned batch with one persistence round trip. + + The caller owns the surrounding transaction. Returned models have their + database-managed identities populated, but are not individually refreshed. + """ + blocks = tuple(_new_block(form) for form in forms) + if not blocks: + return () + logger.info("Creating block batch", extra={"block_count": len(blocks)}) + db_session.add_all(blocks) + db_session.flush() + return blocks + @classmethod async def fetchsert(cls, form: BlockForm, db_session: sqlmodel.Session) -> BlockModel: """Create if not exists, else return the existing one. diff --git a/app/business/info_base/relation.py b/app/business/info_base/relation.py index 3763e9f..67db95d 100644 --- a/app/business/info_base/relation.py +++ b/app/business/info_base/relation.py @@ -5,7 +5,7 @@ from app.engine import SessionLocal from libs.obsrv.main import get_logger from app.schemas.info_base.block import BlockID -from app.schemas.info_base.relation import RelationModel +from app.schemas.info_base.relation import RelationCreateForm, RelationModel from app.schemas.info_base.relation import RelationID from utils.types_ import Undefined, _undefined @@ -122,6 +122,21 @@ def create( ) return relation + @classmethod + def create_many( + cls, + forms: typing.Iterable[RelationCreateForm], + db_session: sqlmodel.Session, + ) -> tuple[RelationModel, ...]: + """Create a caller-owned batch with one persistence round trip.""" + relations = tuple(RelationModel.model_validate(form) for form in forms) + if not relations: + return () + logger.info("Creating relation batch", extra={"relation_count": len(relations)}) + db_session.add_all(relations) + db_session.flush() + return relations + @classmethod def fetchsert( cls, relation: RelationModel, db_session: sqlmodel.Session diff --git a/extensions/github/CHANGELOG.md b/extensions/github/CHANGELOG.md index 9dc5c09..3d40cc4 100644 --- a/extensions/github/CHANGELOG.md +++ b/extensions/github/CHANGELOG.md @@ -4,6 +4,10 @@ This changelog records notable changes to this first-party Python Distribution association. It is generated by [Changie](https://changie.dev/). +## 0.2.0 - 2026-08-24 +### Added +* Synchronize the authenticated account's Stars, Lists, memberships, and repository ownership as reusable graph facts. + ## 0.1.0 - 2026-08-17 ### Changed diff --git a/extensions/github/README.md b/extensions/github/README.md index 6baf29f..d903cbb 100644 --- a/extensions/github/README.md +++ b/extensions/github/README.md @@ -1,41 +1,36 @@ # GitHub Extension for InKCre -This extension provides GitHub Stars source functionality for InKCre. +The GitHub Extension synchronizes the authenticated account's current Stars and GitHub Lists into the InKCre info-base。 -## Features +## Collected graph -- Collect starred repositories from GitHub -- Track repository metadata (stars, forks, languages, topics) -- Automatic tracking of processed stars -- Support for incremental updates - -## Configuration - -The extension requires the following configuration: - -- `github_token`: GitHub personal access token for API access - - Create one at https://github.com/settings/tokens - - Required scopes: `public_repo` (or `repo` for private starred repos) -- `username`: GitHub username to fetch starred repos for -- `include_private`: Whether to include private repositories (default: `false`) +```text +Source --collects--> GitHub Account +GitHub Account --stars--> Repository +GitHub Account --owns--> GitHub List +GitHub List --contains--> Repository +GitHub Account --owns--> Repository +``` -## Usage +Repository、Account and List metadata remain reusable Blocks。When a Star or List membership disappears remotely，ordinary +collection removes the corresponding Relation without deleting those Blocks。 -1. Install the extension in the InKCre database -2. Configure GitHub settings with your token and username -3. Create a GitHub Stars source via the API endpoint: `POST /github/stars` -4. Stars will be collected based on the configured schedule +## Configuration -## Dependencies +Create a Source of type `extensions.github.stars.Source` with： -This extension requires the `PyGithub` package: -```bash -pip install PyGithub +```json +{ + "github_token": "" +} ``` -## Notes +The token determines the authenticated account and visible data。The Source does not accept a separate username or private +repository filter。Changing credentials for the same account is supported；credentials that resolve to another account require +a new Source。 + +## Collection -- The extension tracks the last processed star ID to avoid duplicates -- Repository data includes owner information, description, topics, and statistics -- Respects GitHub API rate limits with built-in delays -- For accessing private starred repositories, ensure your token has appropriate permissions +Dispatch the ordinary `core.source.collect.v1` Job through the generic Source/Job surface。Each run fetches a complete current +Stars and Lists snapshot before reconciling graph facts。There is no extension-specific collection endpoint、incremental +cursor、`full` option or historical backfill mode。 diff --git a/extensions/github/__init__.py b/extensions/github/__init__.py index 65bd949..d1c075b 100644 --- a/extensions/github/__init__.py +++ b/extensions/github/__init__.py @@ -1,7 +1,6 @@ -"""GitHub extension for InKCre - provides GitHub Stars source.""" +"""GitHub Stars and Lists collection Extension.""" import sqlmodel -from fastapi import APIRouter from app.business.extension.main import ExtensionBase @@ -16,26 +15,14 @@ class Extension( ext_id="github", config_cls=GithubExtensionConfig, ): - """GitHub extension - provides GitHub Stars source for collecting starred repositories.""" + """Synchronize GitHub Stars and Lists into reusable graph facts.""" @classmethod def _init_resolvers(cls): - """Initialize GitHub resolvers.""" - from .resolver import GithubRepoResolver # noqa: F401 - from .resolver import GithubUserResolver # noqa: F401 + from .resolver import GitHubAccountResolver # noqa: F401 + from .resolver import GitHubListResolver # noqa: F401 + from .resolver import GitHubRepositoryResolver # noqa: F401 @classmethod def _init_sources(cls): - """Initialize GitHub Stars source.""" - from .stars import Source as GithubStarsSource # noqa: F401 - - @classmethod - def _register_apis(cls, router: APIRouter): - """Register API endpoints for GitHub extension.""" - from app.business.source import SourceManager - - router.post("/stars")( - lambda nickname: SourceManager.create( - f"extensions.{cls.__extid__}.stars.Source", nickname - ) - ) + from .stars import Source as GitHubStarsSource # noqa: F401 diff --git a/extensions/github/adapter.py b/extensions/github/adapter.py new file mode 100644 index 0000000..8df490d --- /dev/null +++ b/extensions/github/adapter.py @@ -0,0 +1,324 @@ +"""Async GitHub GraphQL access without info-base dependencies.""" + +from __future__ import annotations + +import typing + +import httpx + +from .schema import ( + GitHubAccount, + GitHubList, + GitHubListFact, + GitHubRepository, + GitHubRepositoryFact, + GitHubSnapshot, +) + + +_REPOSITORY_FIELDS = """ + id + databaseId + nameWithOwner + description + url + homepageUrl + isPrivate + isArchived + primaryLanguage { name } + repositoryTopics(first: 20) { nodes { topic { name } } } + owner { + __typename + id + login + url + avatarUrl + ... on User { databaseId name } + ... on Organization { databaseId name } + } +""" + +_ACCOUNT_AND_STARS_QUERY = f""" +query GitHubStars($cursor: String) {{ + viewer {{ + id + databaseId + login + name + url + avatarUrl + starredRepositories(first: 50, after: $cursor) {{ + nodes {{ {_REPOSITORY_FIELDS} }} + pageInfo {{ hasNextPage endCursor }} + }} + }} +}} +""" + +_LISTS_QUERY = f""" +query GitHubLists($cursor: String) {{ + viewer {{ + lists(first: 100, after: $cursor) {{ + nodes {{ + id + name + description + slug + isPrivate + items(first: 100) {{ + nodes {{ ... on Repository {{ id }} }} + pageInfo {{ hasNextPage endCursor }} + }} + }} + pageInfo {{ hasNextPage endCursor }} + }} + }} +}} +""" + +_LIST_ITEMS_QUERY = f""" +query GitHubListItems($id: ID!, $cursor: String) {{ + node(id: $id) {{ + ... on UserList {{ + items(first: 100, after: $cursor) {{ + nodes {{ ... on Repository {{ id }} }} + pageInfo {{ hasNextPage endCursor }} + }} + }} + }} +}} +""" + +_REPOSITORIES_QUERY = f""" +query GitHubRepositories($ids: [ID!]!) {{ + nodes(ids: $ids) {{ + ... on Repository {{ {_REPOSITORY_FIELDS} }} + }} +}} +""" + + +class GitHubGraphQLError(RuntimeError): + """GitHub did not provide one complete valid GraphQL observation.""" + + +def _object(value: typing.Any, context: str) -> dict[str, typing.Any]: + if not isinstance(value, dict): + raise GitHubGraphQLError(f"GitHub GraphQL omitted {context}") + return value + + +def _page_info(connection: dict[str, typing.Any]) -> tuple[bool, str | None]: + page = _object(connection.get("pageInfo"), "pageInfo") + has_next = page.get("hasNextPage") + cursor = page.get("endCursor") + if not isinstance(has_next, bool): + raise GitHubGraphQLError("GitHub GraphQL returned invalid pageInfo") + if has_next and not isinstance(cursor, str): + raise GitHubGraphQLError("GitHub GraphQL pagination omitted endCursor") + return has_next, cursor if isinstance(cursor, str) else None + + +def _account(value: dict[str, typing.Any], *, viewer: bool = False) -> GitHubAccount: + typename = "User" if viewer else value.get("__typename") + if typename not in {"User", "Organization"}: + raise GitHubGraphQLError("GitHub returned an unsupported account kind") + return GitHubAccount.model_validate( + { + "node_id": value.get("id"), + "database_id": value.get("databaseId"), + "kind": "user" if typename == "User" else "organization", + "login": value.get("login"), + "name": value.get("name"), + "url": value.get("url"), + "avatar_url": value.get("avatarUrl"), + } + ) + + +def _repository(value: dict[str, typing.Any]) -> GitHubRepositoryFact: + owner = _object(value.get("owner"), "Repository.owner") + language = value.get("primaryLanguage") + topic_connection = _object(value.get("repositoryTopics"), "repositoryTopics") + topic_nodes = topic_connection.get("nodes") + if not isinstance(topic_nodes, list): + raise GitHubGraphQLError("GitHub GraphQL returned invalid repository topics") + topics = tuple( + topic["topic"]["name"] + for topic in topic_nodes + if isinstance(topic, dict) + and isinstance(topic.get("topic"), dict) + and isinstance(topic["topic"].get("name"), str) + ) + return GitHubRepositoryFact( + repository=GitHubRepository.model_validate( + { + "node_id": value.get("id"), + "database_id": value.get("databaseId"), + "name_with_owner": value.get("nameWithOwner"), + "description": value.get("description"), + "url": value.get("url"), + "homepage_url": value.get("homepageUrl"), + "primary_language": (language.get("name") if isinstance(language, dict) else None), + "topics": topics, + "is_private": value.get("isPrivate"), + "is_archived": value.get("isArchived"), + } + ), + owner=_account(owner), + ) + + +class GitHubGraphQLAdapter: + """Fetch one complete GitHub Stars/Lists snapshot.""" + + def __init__(self, token: str, *, client: httpx.AsyncClient | None = None): + self._owned_client = client is None + self._client = client or httpx.AsyncClient( + base_url="https://api.github.com", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "User-Agent": "InKCre-GitHub-Extension", + "X-GitHub-Api-Version": "2022-11-28", + }, + timeout=30, + ) + + async def __aenter__(self) -> "GitHubGraphQLAdapter": + return self + + async def __aexit__(self, *_exc: object) -> None: + if self._owned_client: + await self._client.aclose() + + async def _execute( + self, + query: str, + variables: dict[str, typing.Any], + ) -> dict[str, typing.Any]: + response = await self._client.post( + "/graphql", + json={"query": query, "variables": variables}, + ) + response.raise_for_status() + payload = _object(response.json(), "response") + errors = payload.get("errors") + if errors: + messages = [ + error.get("message", "unknown GraphQL error") + for error in errors + if isinstance(error, dict) + ] + raise GitHubGraphQLError("; ".join(messages) or "GitHub GraphQL request failed") + return _object(payload.get("data"), "data") + + async def fetch_snapshot(self) -> GitHubSnapshot: + repositories: dict[str, GitHubRepositoryFact] = {} + starred_ids: list[str] = [] + account: GitHubAccount | None = None + cursor: str | None = None + + while True: + data = await self._execute(_ACCOUNT_AND_STARS_QUERY, {"cursor": cursor}) + viewer = _object(data.get("viewer"), "viewer") + observed_account = _account(viewer, viewer=True) + if account is not None and observed_account.node_id != account.node_id: + raise GitHubGraphQLError("GitHub viewer changed during pagination") + account = observed_account + connection = _object(viewer.get("starredRepositories"), "starredRepositories") + nodes = connection.get("nodes") + if not isinstance(nodes, list): + raise GitHubGraphQLError("GitHub GraphQL returned invalid starredRepositories") + for node in nodes: + fact = _repository(_object(node, "starred Repository")) + repositories[fact.repository.node_id] = fact + starred_ids.append(fact.repository.node_id) + has_next, cursor = _page_info(connection) + if not has_next: + break + + lists: list[GitHubListFact] = [] + cursor = None + while True: + data = await self._execute(_LISTS_QUERY, {"cursor": cursor}) + viewer = _object(data.get("viewer"), "viewer") + connection = _object(viewer.get("lists"), "viewer.lists") + nodes = connection.get("nodes") + if not isinstance(nodes, list): + raise GitHubGraphQLError("GitHub GraphQL returned invalid Lists") + for node_value in nodes: + node = _object(node_value, "List") + item_connection = _object(node.get("items"), "List.items") + item_ids = self._collect_repository_ids(item_connection) + has_more_items, item_cursor = _page_info(item_connection) + while has_more_items: + item_data = await self._execute( + _LIST_ITEMS_QUERY, + {"id": node.get("id"), "cursor": item_cursor}, + ) + list_node = _object(item_data.get("node"), "UserList node") + item_connection = _object(list_node.get("items"), "List.items") + item_ids.extend(self._collect_repository_ids(item_connection)) + has_more_items, item_cursor = _page_info(item_connection) + lists.append( + GitHubListFact( + list=GitHubList.model_validate( + { + "node_id": node.get("id"), + "name": node.get("name"), + "description": node.get("description"), + "slug": node.get("slug"), + "is_private": node.get("isPrivate"), + } + ), + repository_node_ids=tuple(dict.fromkeys(item_ids)), + ) + ) + has_next, cursor = _page_info(connection) + if not has_next: + break + + if account is None: # pragma: no cover - at least one Stars page is required + raise GitHubGraphQLError("GitHub GraphQL omitted the authenticated account") + unknown_memberships = { + node_id + for list_ in lists + for node_id in list_.repository_node_ids + if node_id not in repositories + } + missing_ids = sorted(unknown_memberships) + for offset in range(0, len(missing_ids), 50): + requested = missing_ids[offset : offset + 50] + data = await self._execute(_REPOSITORIES_QUERY, {"ids": requested}) + nodes = data.get("nodes") + if not isinstance(nodes, list): + raise GitHubGraphQLError("GitHub GraphQL returned invalid Repository nodes") + for node in nodes: + fact = _repository(_object(node, "List-only Repository")) + repositories[fact.repository.node_id] = fact + unresolved = unknown_memberships - repositories.keys() + if unresolved: + raise GitHubGraphQLError("GitHub could not resolve every List Repository") + return GitHubSnapshot( + account=account, + repositories=tuple(repositories.values()), + starred_repository_node_ids=tuple(dict.fromkeys(starred_ids)), + lists=tuple(lists), + ) + + @staticmethod + def _collect_repository_ids(connection: dict[str, typing.Any]) -> list[str]: + nodes = connection.get("nodes") + if not isinstance(nodes, list): + raise GitHubGraphQLError("GitHub GraphQL returned invalid List items") + result: list[str] = [] + for node in nodes: + node_id = _object(node, "List Repository").get("id") + if not isinstance(node_id, str): + raise GitHubGraphQLError("GitHub List Repository omitted its node ID") + result.append(node_id) + return result + + +__all__ = ["GitHubGraphQLAdapter", "GitHubGraphQLError"] diff --git a/extensions/github/pyproject.toml b/extensions/github/pyproject.toml index 1c6a8c1..fbdd5c7 100644 --- a/extensions/github/pyproject.toml +++ b/extensions/github/pyproject.toml @@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta" [project] name = "inkcre-ext-github" -version = "0.1.0" +version = "0.2.0" description = "GitHub extension for InKCre" authors = [{ name = "Lan_zhijiang", email = "lanzhijiang@foxmail.com" }] dependencies = [ "fastapi>=0.139.2,<0.140.0", "pydantic>=2.11.7,<3.0.0", - "PyGithub>=2.8.1,<3.0.0", + "httpx>=0.28.1,<0.29.0", "sqlmodel>=0.0.24,<0.0.25", ] requires-python = ">=3.12,<3.13" diff --git a/extensions/github/repository.py b/extensions/github/repository.py new file mode 100644 index 0000000..c921f87 --- /dev/null +++ b/extensions/github/repository.py @@ -0,0 +1,344 @@ +"""Transactional reconciliation of one complete GitHub snapshot.""" + +from __future__ import annotations + +import typing + +import pydantic +import sqlalchemy +import sqlmodel + +from app.business.info_base.block import BlockManager +from app.business.info_base.relation import RelationManager +from app.business.source import SourceManager +from app.schemas.info_base.block import BlockModel +from app.schemas.info_base.relation import RelationCreateForm, RelationModel +from app.schemas.source import SourceModel + +from .resolver import ( + ACCOUNT_RESOLVER_ID, + LIST_RESOLVER_ID, + REPOSITORY_RESOLVER_ID, + GitHubAccountResolver, + GitHubGraphIntegrityError, + GitHubListResolver, + GitHubRepositoryResolver, +) +from .schema import ( + GitHubAccount, + GitHubList, + GitHubRepository, + GitHubSnapshot, + GitHubSourceState, +) + + +class GitHubSourceBindingError(RuntimeError): + """A Source token resolved to another GitHub Account.""" + + +class GitHubReconcileReport(pydantic.BaseModel): + """Bounded observable effect summary for one complete snapshot.""" + + model_config = pydantic.ConfigDict(extra="forbid") + account: str + stars: int + lists: int + memberships: int + blocks_created: int = 0 + blocks_updated: int = 0 + relations_created: int = 0 + relations_deleted: int = 0 + + +def _id(block: BlockModel) -> int: + if block.id is None: # pragma: no cover - database persistence invariant + raise RuntimeError("Persisted GitHub Block has no ID") + return block.id + + +class GitHubGraphRepository: + """Own exact GitHub identity and current relation-set reconciliation.""" + + _resolver_models: typing.ClassVar[dict[str, type[pydantic.BaseModel]]] = { + ACCOUNT_RESOLVER_ID: GitHubAccount, + REPOSITORY_RESOLVER_ID: GitHubRepository, + LIST_RESOLVER_ID: GitHubList, + } + + def __init__(self, db_session: sqlmodel.Session): + self.db = db_session + self.blocks_created = 0 + self.blocks_updated = 0 + self.relations_created = 0 + self.relations_deleted = 0 + + def reconcile(self, source_id: int, snapshot: GitHubSnapshot) -> GitHubReconcileReport: + source = self.db.exec( + sqlmodel.select(SourceModel).where(SourceModel.id == source_id).with_for_update() + ).one() + state = GitHubSourceState.model_validate(source.state or {}) + if ( + state.account_node_id is not None + and state.account_node_id != snapshot.account.node_id + ): + raise GitHubSourceBindingError( + "GitHub Source token resolves to a different Account; create another Source" + ) + + source_anchor = SourceManager.ensure_block(source, self.db) + existing = self._load_github_blocks() + repositories = { + fact.repository.node_id: fact.repository for fact in snapshot.repositories + } + accounts = {fact.owner.node_id: fact.owner for fact in snapshot.repositories} + accounts[snapshot.account.node_id] = snapshot.account + lists = {fact.list.node_id: fact.list for fact in snapshot.lists} + + repository_blocks = self._upsert_many(GitHubRepositoryResolver, repositories, existing) + account_blocks = self._upsert_many(GitHubAccountResolver, accounts, existing) + list_blocks = self._upsert_many(GitHubListResolver, lists, existing) + + source_block_id = _id(source_anchor) + account_block_id = _id(account_blocks[snapshot.account.node_id]) + repository_ids = {node_id: _id(block) for node_id, block in repository_blocks.items()} + current_list_ids = {node_id: _id(block) for node_id, block in list_blocks.items()} + previous_list_ids = self._previous_list_ids(account_block_id, existing) + roots = { + source_block_id, + account_block_id, + *previous_list_ids, + *current_list_ids.values(), + } + candidates = self._load_candidate_relations(roots, set(repository_ids.values())) + + blocks_by_id = {_id(block): block for block in existing.values()} + for values in ( + repository_blocks.values(), + account_blocks.values(), + list_blocks.values(), + ): + blocks_by_id.update({_id(block): block for block in values}) + desired = self._desired_relations( + source_block_id, + account_block_id, + repository_ids, + account_blocks, + current_list_ids, + snapshot, + ) + managed = self._managed_relations( + candidates, + blocks_by_id, + source_block_id, + account_block_id, + previous_list_ids | set(current_list_ids.values()), + set(repository_ids.values()), + ) + self._replace_relations(managed, desired) + + source.state = GitHubSourceState(account_node_id=snapshot.account.node_id).model_dump( + mode="json" + ) + self.db.add(source) + self.db.flush() + return GitHubReconcileReport( + account=snapshot.account.login, + stars=len(snapshot.starred_repository_node_ids), + lists=len(snapshot.lists), + memberships=sum(len(item.repository_node_ids) for item in snapshot.lists), + blocks_created=self.blocks_created, + blocks_updated=self.blocks_updated, + relations_created=self.relations_created, + relations_deleted=self.relations_deleted, + ) + + def _load_github_blocks(self) -> dict[tuple[str, str], BlockModel]: + blocks = self.db.exec( + sqlmodel.select(BlockModel).where( + BlockModel.resolver.in_(tuple(self._resolver_models)) # type: ignore[union-attr] + ) + ).all() + indexed: dict[tuple[str, str], BlockModel] = {} + for block in blocks: + resolver_id = block.resolver + try: + content = self._resolver_models[resolver_id].model_validate_json(block.content) + node_id = typing.cast(str, getattr(content, "node_id")) + except (pydantic.ValidationError, TypeError, KeyError) as error: + raise GitHubGraphIntegrityError( + f"GitHub Block {_id(block)} has malformed canonical content" + ) from error + key = (resolver_id, node_id) + if key in indexed: + raise GitHubGraphIntegrityError( + f"GitHub node {node_id!r} resolves to multiple Blocks" + ) + indexed[key] = block + return indexed + + def _upsert_many( + self, + resolver_cls: typing.Any, + contents: typing.Mapping[str, GitHubAccount | GitHubRepository | GitHubList], + existing: dict[tuple[str, str], BlockModel], + ) -> dict[str, BlockModel]: + resolver_id = typing.cast(str, resolver_cls.__rsotype__) + result: dict[str, BlockModel] = {} + missing: list[tuple[str, typing.Any]] = [] + for node_id, content in contents.items(): + form = resolver_cls.create_block(content) + block = existing.get((resolver_id, node_id)) + if block is None: + missing.append((node_id, form)) + else: + if block.content != form.content or block.storage is not None: + block.storage = None + block.content = form.content + self.db.add(block) + self.blocks_updated += 1 + result[node_id] = block + created = BlockManager.create_many((form for _, form in missing), self.db) + self.blocks_created += len(created) + for (node_id, _), block in zip(missing, created, strict=True): + result[node_id] = block + existing[(resolver_id, node_id)] = block + return result + + def _previous_list_ids( + self, account_id: int, blocks: dict[tuple[str, str], BlockModel] + ) -> set[int]: + list_ids = { + _id(block) for (resolver, _), block in blocks.items() if resolver == LIST_RESOLVER_ID + } + if not list_ids: + return set() + return set( + self.db.exec( + sqlmodel.select(RelationModel.to_).where( + RelationModel.from_ == account_id, + RelationModel.content == "owns", + RelationModel.to_.in_(tuple(list_ids)), # type: ignore[union-attr] + ) + ).all() + ) + + def _load_candidate_relations( + self, roots: set[int], repository_ids: set[int] + ) -> tuple[RelationModel, ...]: + endpoints = roots | repository_ids + if not endpoints: + return () + return tuple( + self.db.exec( + sqlmodel.select(RelationModel).where( + RelationModel.content.in_(("collects", "stars", "owns", "contains")), # type: ignore[union-attr] + sqlalchemy.or_( + RelationModel.from_.in_(tuple(endpoints)), # type: ignore[union-attr] + RelationModel.to_.in_(tuple(endpoints)), # type: ignore[union-attr] + ), + ) + ).all() + ) + + @staticmethod + def _desired_relations( # noqa: PLR0913 + source_block_id: int, + account_block_id: int, + repository_ids: dict[str, int], + account_blocks: dict[str, BlockModel], + list_ids: dict[str, int], + snapshot: GitHubSnapshot, + ) -> set[tuple[int, int, str]]: + desired = {(source_block_id, account_block_id, "collects")} + desired.update( + (account_block_id, repository_ids[node_id], "stars") + for node_id in snapshot.starred_repository_node_ids + ) + desired.update((account_block_id, list_id, "owns") for list_id in list_ids.values()) + desired.update( + (list_ids[item.list.node_id], repository_ids[node_id], "contains") + for item in snapshot.lists + for node_id in item.repository_node_ids + ) + desired.update( + ( + _id(account_blocks[item.owner.node_id]), + repository_ids[item.repository.node_id], + "owns", + ) + for item in snapshot.repositories + ) + return desired + + @staticmethod + def _managed_relations( # noqa: PLR0913 + relations: tuple[RelationModel, ...], + blocks_by_id: dict[int, BlockModel], + source_block_id: int, + account_block_id: int, + list_ids: set[int], + current_repository_ids: set[int], + ) -> tuple[RelationModel, ...]: + managed: list[RelationModel] = [] + for relation in relations: + from_block = blocks_by_id.get(relation.from_) + to_block = blocks_by_id.get(relation.to_) + if ( + ( + relation.content == "collects" + and relation.from_ == source_block_id + and to_block is not None + and to_block.resolver == ACCOUNT_RESOLVER_ID + ) + or ( + relation.content == "stars" + and relation.from_ == account_block_id + and to_block is not None + and to_block.resolver == REPOSITORY_RESOLVER_ID + ) + or ( + relation.content == "owns" + and relation.from_ == account_block_id + and to_block is not None + and to_block.resolver == LIST_RESOLVER_ID + ) + or ( + relation.content == "contains" + and relation.from_ in list_ids + and to_block is not None + and to_block.resolver == REPOSITORY_RESOLVER_ID + ) + or ( + relation.content == "owns" + and relation.to_ in current_repository_ids + and from_block is not None + and from_block.resolver == ACCOUNT_RESOLVER_ID + ) + ): + managed.append(relation) + return tuple(managed) + + def _replace_relations( + self, existing: tuple[RelationModel, ...], desired: set[tuple[int, int, str]] + ) -> None: + retained: set[tuple[int, int, str]] = set() + for relation in existing: + key = (relation.from_, relation.to_, relation.content) + if key in desired and key not in retained: + retained.add(key) + else: + self.db.delete(relation) + self.relations_deleted += 1 + missing = desired - retained + RelationManager.create_many( + ( + RelationCreateForm(from_=from_, to_=to_, content=content) + for from_, to_, content in missing + ), + self.db, + ) + self.relations_created += len(missing) + + +__all__ = ["GitHubGraphRepository", "GitHubReconcileReport", "GitHubSourceBindingError"] diff --git a/extensions/github/resolver.py b/extensions/github/resolver.py index 584599d..aaa8fa9 100644 --- a/extensions/github/resolver.py +++ b/extensions/github/resolver.py @@ -1,85 +1,95 @@ -"""GitHub resolver for handling GitHub blocks.""" +"""Canonical GitHub Block producers and use-time projections.""" -from typing import Optional as Opt +from __future__ import annotations + +import typing -from sqlmodel import Session import sqlmodel + from app.business.info_base.resolver import Resolver, TextProjectionContext from app.business.info_base.resolver.label import format_label from app.schemas.info_base.block import BlockForm, BlockModel +from app.schemas.info_base.main import InArcForm, OutArcForm, StarsGraphForm from app.schemas.info_base.relation import RelationForm -from app.schemas.info_base.main import InArcForm, StarsGraphForm -from utils.sql import find_by_json_contains -from .schema import GithubRepo, GithubUser +from utils.sql import find_by_json_field +from .schema import GitHubAccount, GitHubList, GitHubRepository -class GithubRepoResolver( - Resolver[GithubRepo, str], - rso_type="extensions.github.repo.v1", -): - """Resolver for GitHub repository blocks.""" + +ACCOUNT_RESOLVER_ID = "extensions.github.account.v1" +REPOSITORY_RESOLVER_ID = "extensions.github.repository.v1" +LIST_RESOLVER_ID = "extensions.github.list.v1" + + +class GitHubGraphIntegrityError(RuntimeError): + """Exact GitHub graph identity is ambiguous or malformed.""" + + +class _GitHubResolverMixin: + content_model: typing.ClassVar[type[GitHubAccount | GitHubRepository | GitHubList]] def __post_init__(self, raw_content: str | None = None) -> None: - if raw_content is None: - raise ValueError("GitHub repository blocks require inline JSON content") - self._content = GithubRepo.model_validate_json(raw_content) - self.set_solved_content(self._content) + if raw_content is not None: + resolver = typing.cast(Resolver[typing.Any, str], self) + resolver.set_solved_content(self.content_model.model_validate_json(raw_content)) async def _get_solved_content( self, *, refresh: bool = False, materialize_missing: bool = True, - ) -> GithubRepo: + ) -> typing.Any: del materialize_missing - self._content = GithubRepo.model_validate_json( - await self.get_raw_content(refresh=refresh) + resolver = typing.cast(Resolver[typing.Any, str], self) + return self.content_model.model_validate_json( + await resolver.get_raw_content(refresh=refresh) ) - return self._content @classmethod - def create_graph( - cls, - repo: GithubRepo, - owner: Opt[GithubUser] = None, - ) -> StarsGraphForm: - """Create a StarGraphForm from GitHub repository data. - - :param repo: GitHub repository - :param owner: Repository owner (GitHub user), optional - :return: StarGraphForm representing the repository graph - ```mermaid - graph TD - A[GitHub User] -->|owns| B[GitHub Repo] - ``` - """ - in_relations = () - if owner: - in_relations = ( - InArcForm( - relation=RelationForm(content="owns"), - from_graph=GithubUserResolver.create_graph(owner), - ), - ) - - return StarsGraphForm( - in_arcs=in_relations, - block=BlockForm( - resolver=cls.__rsotype__, - content=repo.model_dump_json(), - ), - out_arcs=(), + def create_block(cls, content, storage=None) -> BlockForm: + canonical = cls.content_model.model_validate(content) + return BlockForm( + resolver=typing.cast(typing.Any, cls).__rsotype__, + content=canonical.model_dump_json(), + storage=storage, ) - def get_existing(self, db_session: Session) -> BlockModel | None: - """Check for existing GitHub repo by ID.""" - existing_block = db_session.exec( + @classmethod + def create_graph(cls, content) -> StarsGraphForm: + return StarsGraphForm(block=cls.create_block(content)) + + @classmethod + def find_existing( + cls, + node_id: str, + db_session: sqlmodel.Session, + ) -> BlockModel | None: + matches = db_session.exec( sqlmodel.select(BlockModel).where( - BlockModel.resolver == self._block.resolver, - find_by_json_contains(BlockModel.content, {"id": self._content.id}), + BlockModel.resolver == typing.cast(typing.Any, cls).__rsotype__, + find_by_json_field(BlockModel.content, "node_id", node_id), ) - ).one_or_none() - return existing_block + ).all() + if len(matches) > 1: + raise GitHubGraphIntegrityError( + f"GitHub node {node_id!r} resolves to multiple Blocks" + ) + return matches[0] if matches else None + + def get_existing(self, db_session: sqlmodel.Session) -> BlockModel | None: + resolver = typing.cast(Resolver[typing.Any, str], self) + content = self.content_model.model_validate_json(resolver._block.content) + return self.find_existing(content.node_id, db_session) + + +class GitHubAccountResolver( + _GitHubResolverMixin, + Resolver[GitHubAccount, str], + rso_type=ACCOUNT_RESOLVER_ID, +): + """Resolve one GitHub user or organization account.""" + + content_model = GitHubAccount async def get_text( self, @@ -88,64 +98,112 @@ async def get_text( refresh: bool = False, materialize_missing: bool = True, ) -> str: - """Return one complete reusable textual projection of the repository.""" del context - content = await self.get_solved_content( + account = await self.get_solved_content( refresh=refresh, materialize_missing=materialize_missing, ) - parts = [content.full_name] - if content.description: - parts.append(content.description) - if content.language: - parts.append(f"Language: {content.language}") - if content.topics: - parts.append(f"Topics: {', '.join(content.topics)}") - return "\n".join(parts) + return f"{account.name} (@{account.login})" if account.name else f"@{account.login}" async def get_label(self, *, refresh: bool = False) -> str: - content = await self.get_solved_content( - refresh=refresh, - materialize_missing=False, - ) - return format_label("github repository", content.full_name) + account = GitHubAccount.model_validate_json(await self.get_raw_content(refresh=refresh)) + return format_label(f"github {account.kind}", account.login) -class GithubUserResolver( - Resolver[GithubUser, str], - rso_type="extensions.github.user.v1", +class GitHubRepositoryResolver( + _GitHubResolverMixin, + Resolver[GitHubRepository, str], + rso_type=REPOSITORY_RESOLVER_ID, ): - """Resolver for GitHub user blocks.""" + """Resolve one GitHub Repository.""" - def __post_init__(self, raw_content: str | None = None) -> None: - if raw_content is None: - raise ValueError("GitHub user blocks require inline JSON content") - self._content = GithubUser.model_validate_json(raw_content) - self.set_solved_content(self._content) + content_model = GitHubRepository - async def _get_solved_content( + @classmethod + def create_graph( + cls, + content: GitHubRepository, + owner: GitHubAccount | None = None, + ) -> StarsGraphForm: + incoming = ( + ( + InArcForm( + relation=RelationForm(content="owns"), + from_graph=GitHubAccountResolver.create_graph(owner), + ), + ) + if owner is not None + else () + ) + return StarsGraphForm(block=cls.create_block(content), in_arcs=incoming) + + async def get_text( self, *, + context: TextProjectionContext = "default", refresh: bool = False, materialize_missing: bool = True, - ) -> GithubUser: - del materialize_missing - self._content = GithubUser.model_validate_json( - await self.get_raw_content(refresh=refresh) + ) -> str: + del context + repository = await self.get_solved_content( + refresh=refresh, + materialize_missing=materialize_missing, ) - return self._content + parts = [repository.name_with_owner] + if repository.description: + parts.append(repository.description) + if repository.primary_language: + parts.append(f"Language: {repository.primary_language}") + if repository.topics: + parts.append(f"Topics: {', '.join(repository.topics)}") + return "\n".join(parts) - @classmethod - def create_block(cls, content: GithubUser | dict, storage=None) -> BlockForm: - return BlockForm( - resolver=cls.__rsotype__, - content=GithubUser.model_validate(content).model_dump_json(), - storage=storage, + async def get_label(self, *, refresh: bool = False) -> str: + repository = GitHubRepository.model_validate_json( + await self.get_raw_content(refresh=refresh) ) + return format_label("github repository", repository.name_with_owner) + + +class GitHubListResolver( + _GitHubResolverMixin, + Resolver[GitHubList, str], + rso_type=LIST_RESOLVER_ID, +): + """Resolve one GitHub List independently of current membership.""" + + content_model = GitHubList @classmethod - def create_graph(cls, user: GithubUser | dict) -> StarsGraphForm: - return StarsGraphForm(block=cls.create_block(user)) + def create_graph( + cls, + content: GitHubList, + *, + owner: GitHubAccount | None = None, + repositories: typing.Iterable[GitHubRepository] = (), + ) -> StarsGraphForm: + incoming = ( + ( + InArcForm( + relation=RelationForm(content="owns"), + from_graph=GitHubAccountResolver.create_graph(owner), + ), + ) + if owner is not None + else () + ) + outgoing = tuple( + OutArcForm( + relation=RelationForm(content="contains"), + to_graph=GitHubRepositoryResolver.create_graph(repository), + ) + for repository in repositories + ) + return StarsGraphForm( + block=cls.create_block(content), + in_arcs=incoming, + out_arcs=outgoing, + ) async def get_text( self, @@ -154,32 +212,24 @@ async def get_text( refresh: bool = False, materialize_missing: bool = True, ) -> str: - """Get text representation of the GitHub user. - - Returns the display name and login, or just login if no name. - """ del context - content = await self.get_solved_content( + list_ = await self.get_solved_content( refresh=refresh, materialize_missing=materialize_missing, ) - if content.name: - return f"{content.name} (@{content.login})" - return f"@{content.login}" + return "\n".join(part for part in (list_.name, list_.description) if part) async def get_label(self, *, refresh: bool = False) -> str: - content = await self.get_solved_content( - refresh=refresh, - materialize_missing=False, - ) - return format_label("github user", content.login) - - def get_existing(self, db_session: Session) -> BlockModel | None: - """Check for existing GitHub user by ID.""" - existing_block = db_session.exec( - sqlmodel.select(BlockModel).where( - BlockModel.resolver == self._block.resolver, - find_by_json_contains(BlockModel.content, {"id": self._content.id}), - ) - ).one_or_none() - return existing_block + list_ = GitHubList.model_validate_json(await self.get_raw_content(refresh=refresh)) + return format_label("github list", list_.name) + + +__all__ = [ + "ACCOUNT_RESOLVER_ID", + "GitHubAccountResolver", + "GitHubGraphIntegrityError", + "GitHubListResolver", + "GitHubRepositoryResolver", + "LIST_RESOLVER_ID", + "REPOSITORY_RESOLVER_ID", +] diff --git a/extensions/github/schema.py b/extensions/github/schema.py index 1c7811f..9bb365d 100644 --- a/extensions/github/schema.py +++ b/extensions/github/schema.py @@ -1,69 +1,107 @@ -"""InKCre GitHub Extension Schemas.""" +"""Canonical GitHub collection facts.""" -__all__ = [ - "GithubUser", - "GithubRepo", -] +from __future__ import annotations + +import typing -from datetime import datetime -from typing import Optional as Opt -import sqlmodel import pydantic -class GithubUser(sqlmodel.SQLModel): - """GitHub user.""" +class GitHubAccount(pydantic.BaseModel): + """One canonical GitHub user or organization account.""" + model_config = pydantic.ConfigDict(extra="forbid") + + node_id: str + database_id: int | None = None + kind: typing.Literal["user", "organization"] login: str - """GitHub username (login)""" - id: int - """GitHub user ID""" - name: Opt[str] = None - """Display name of the user""" - avatar_url: Opt[str] = None - """URL to user's avatar""" - html_url: str - """URL to user's GitHub profile""" - - @pydantic.field_validator("login") - @classmethod - def normalize_login(cls, v: str) -> str: - """Normalize GitHub login to lowercase.""" - return v.lower().strip() - - -class GithubRepo(sqlmodel.SQLModel): - """GitHub repository block content model.""" - - id: int - """Repository ID from GitHub""" + name: str | None = None + url: str + avatar_url: str | None = None + + +class GitHubRepository(pydantic.BaseModel): + """Stable Repository identity and useful source-authored metadata.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + node_id: str + database_id: int | None = None + name_with_owner: str + description: str | None = None + url: str + homepage_url: str | None = None + primary_language: str | None = None + topics: tuple[str, ...] = () + is_private: bool + is_archived: bool + + +class GitHubList(pydantic.BaseModel): + """One GitHub List independent of its current membership.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + node_id: str name: str - """Repository name (e.g., 'core-py')""" - full_name: str - """Full repository name (e.g., 'InKCre/core-py')""" - description: Opt[str] = None - """Repository description""" - html_url: str - """URL to repository on GitHub""" - homepage: Opt[str] = None - """Repository homepage URL""" - language: Opt[str] = None - """Primary programming language""" - stargazers_count: int = 0 - """Number of stars""" - watchers_count: int = 0 - """Number of watchers""" - forks_count: int = 0 - """Number of forks""" - open_issues_count: int = 0 - """Number of open issues""" - topics: list[str] = [] - """Repository topics/tags""" - created_at: datetime - """Repository creation time""" - updated_at: datetime - """Repository last update time""" - pushed_at: Opt[datetime] = None - """Last push time""" - starred_at: Opt[datetime] = None - """When the user starred this repo""" + description: str | None = None + slug: str + is_private: bool + + +class GitHubRepositoryFact(pydantic.BaseModel): + """One Repository together with its current canonical owner.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + repository: GitHubRepository + owner: GitHubAccount + + +class GitHubListFact(pydantic.BaseModel): + """One List and the exact Repository identities visible in its snapshot.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + list: GitHubList + repository_node_ids: tuple[str, ...] = () + + +class GitHubSnapshot(pydantic.BaseModel): + """One complete authenticated Stars and Lists observation.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + account: GitHubAccount + repositories: tuple[GitHubRepositoryFact, ...] = () + starred_repository_node_ids: tuple[str, ...] = () + lists: tuple[GitHubListFact, ...] = () + + +class GitHubSourceConfig(pydantic.BaseModel): + """Credentials for one GitHub access context.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + github_token: str = pydantic.Field(min_length=1) + + +class GitHubSourceState(pydantic.BaseModel): + """Stable external account binding accepted by one Source.""" + + model_config = pydantic.ConfigDict(extra="forbid") + + account_node_id: str | None = None + + +__all__ = [ + "GitHubAccount", + "GitHubList", + "GitHubListFact", + "GitHubRepository", + "GitHubRepositoryFact", + "GitHubSnapshot", + "GitHubSourceConfig", + "GitHubSourceState", +] diff --git a/extensions/github/stars.py b/extensions/github/stars.py index cec610b..e5d4a4c 100644 --- a/extensions/github/stars.py +++ b/extensions/github/stars.py @@ -1,223 +1,31 @@ -"""GitHub Stars Source for collecting starred repositories.""" +"""GitHub Stars and Lists Source orchestration.""" -import asyncio +from __future__ import annotations import pydantic -import sqlmodel + from app.business.source import SourceBase from app.engine import SessionLocal -from app.business.info_base.main import InfoBaseManager -from app.schemas.info_base.main import StarsGraphForm from app.schemas.job import JobModel -from extensions.github.resolver import GithubRepoResolver -from libs.obsrv.main import get_logger -from .schema import GithubRepo, GithubUser - -LOGGER = get_logger().getChild(__name__) - - -class SourceConfig(sqlmodel.SQLModel): - """Configuration of GitHub Stars Source.""" - - github_token: str = "" - """GitHub personal access token for API access""" - username: str = "" - """GitHub username to fetch starred repos for""" - include_private: bool = False - """Whether to include private repositories (requires appropriate token permissions)""" - - -class CollectConfig(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - full: bool = False +from .adapter import GitHubGraphQLAdapter +from .repository import GitHubGraphRepository +from .schema import GitHubSourceConfig -class Source( - SourceBase[SourceConfig], - config_cls=SourceConfig, - collect_config_cls=CollectConfig, -): - """GitHub Stars Source - collects starred repositories from GitHub.""" +class Source(SourceBase[GitHubSourceConfig], config_cls=GitHubSourceConfig): + """Synchronize the authenticated GitHub Account's Stars and Lists.""" async def collect(self, job: JobModel, config: pydantic.BaseModel) -> None: - """Collect starred repositories from GitHub. - - By default, collects new stars since last collection. - If this is the first run or 'full' is specified in job config, collects all stars. - """ - logger = LOGGER.getChild(f"collect.{job.id}") - config = self.get_config() - collect_config = CollectConfig.model_validate(config) - full = collect_config.full - - logger.info( - "Starting GitHub stars collection", - extra={"job_id": job.id, "source": self._id, "full": full}, - ) - - # Import PyGithub - try: - from github import Github, GithubException - except ImportError: - logger.error("PyGithub not installed. Please install with: pip install PyGithub") - raise ImportError("PyGithub is required for GitHub Stars source") - - # Initialize GitHub client - try: - gh = Github(config.github_token) - # Verify authentication - user = gh.get_user(config.username) - logger.info( - "Connected to GitHub", - extra={"username": config.username}, - ) - except GithubException as e: - logger.error( - "Failed to authenticate with GitHub", - extra={"username": config.username, "error": str(e)}, - exc_info=True, - ) - raise e - - collected: list[StarsGraphForm] = [] - try: - # Get starred repositories - try: - starred = user.get_starred() - logger.info("Fetching starred repositories") - except GithubException as e: - logger.error( - "Failed to fetch starred repositories", - extra={"error": str(e)}, - exc_info=True, - ) - raise e - - # Get state for tracking - state = self.get_state() - last_starred_id = state.get("last_starred_id") - - # Process starred repositories - processed_count = 0 - new_stars_count = 0 - reached_last_star = False - - for starred_repo in starred: - processed_count += 1 - - # Stop if we've reached the last collected star (not in full mode) - if not full and last_starred_id and starred_repo.id == last_starred_id: - logger.info( - "Reached last collected star, stopping", - extra={"repo_id": starred_repo.id}, - ) - reached_last_star = True - break - - # Skip private repos if not configured to include them - if starred_repo.private and not config.include_private: - logger.debug( - "Skipping private repository", - extra={"repo": starred_repo.full_name}, - ) - continue - - logger.info( - "Processing starred repository", - extra={"repo": starred_repo.full_name, "count": processed_count}, - ) - - # Extract repository data - try: - # Note: starred_at timestamp requires GitHub's Star API which is not - # directly available in PyGithub's starred repos. We use None here. - starred_at = None - - repo = GithubRepo( - id=starred_repo.id, - name=starred_repo.name, - full_name=starred_repo.full_name, - description=starred_repo.description, - html_url=starred_repo.html_url, - homepage=starred_repo.homepage, - language=starred_repo.language, - stargazers_count=starred_repo.stargazers_count, - watchers_count=starred_repo.watchers_count, - forks_count=starred_repo.forks_count, - open_issues_count=starred_repo.open_issues_count, - topics=starred_repo.get_topics(), - created_at=starred_repo.created_at, - updated_at=starred_repo.updated_at, - pushed_at=starred_repo.pushed_at, - starred_at=starred_at, - ) - - # Extract owner data (owner can be None for deleted accounts) - owner = None - if starred_repo.owner: - owner = GithubUser( - login=starred_repo.owner.login, - id=starred_repo.owner.id, - name=starred_repo.owner.name if starred_repo.owner.name else None, - avatar_url=starred_repo.owner.avatar_url, - html_url=starred_repo.owner.html_url, - ) - - # Create graph - collected.append(GithubRepoResolver.create_graph(repo, owner)) - new_stars_count += 1 - - # Update state to track the most recent star - if processed_count == 1: - state["last_starred_id"] = starred_repo.id - self.set_state(state) - logger.debug("Updated last starred ID", extra={"repo_id": starred_repo.id}) - - logger.info( - "Collected starred repository", - extra={ - "repo": starred_repo.full_name, - "stars": starred_repo.stargazers_count, - }, - ) - - except Exception as e: - logger.warning( - "Failed to process starred repository", - extra={"repo": starred_repo.full_name, "error": str(e)}, - exc_info=True, - ) - continue + del config + source_config = self.get_config() + async with GitHubGraphQLAdapter(source_config.github_token) as adapter: + snapshot = await adapter.fetch_snapshot() - # Small delay to respect rate limits - await asyncio.sleep(0.1) + with SessionLocal() as db_session: + report = GitHubGraphRepository(db_session).reconcile(self._id, snapshot) + db_session.commit() + job.state = report.model_dump(mode="json") - finally: - # PyGithub doesn't require explicit connection closing - logger.info("GitHub collection session ended") - logger.info( - "Saving collected stars to database", - extra={"count": len(collected), "new_stars": new_stars_count}, - ) - try: - with SessionLocal() as db: - for graph in collected: - await InfoBaseManager.add_stars_graph_to_session(graph, db) - db.commit() - logger.info( - "GitHub stars collection completed", - extra={ - "job_id": job.id, - "stars_collected": len(collected), - "processed": processed_count, - }, - ) - except Exception as e: - logger.error( - "Failed to save stars to database", - extra={"job_id": job.id, "error": str(e)}, - exc_info=True, - ) - raise e +__all__ = ["Source"] diff --git a/pdm.lock b/pdm.lock index a3aead8..bde6c73 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "extension-preview", "extension-publisher"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:8d0bbdbea4f86221f6bec2534ef497777b44fd422bf75bcbeb5597a30b40aa72" +content_hash = "sha256:c67bbf838738d22709dd73802bed614471087e1129f43bfa412de0a15a2c43a9" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -266,32 +266,6 @@ files = [ {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] -[[package]] -name = "cffi" -version = "2.1.1" -requires_python = ">=3.10" -summary = "Foreign Function Interface for Python calling C code." -groups = ["default"] -marker = "platform_python_implementation != \"PyPy\"" -dependencies = [ - "pycparser; implementation_name != \"PyPy\"", -] -files = [ - {file = "cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0"}, - {file = "cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf"}, - {file = "cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a"}, - {file = "cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890"}, - {file = "cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50"}, - {file = "cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e"}, - {file = "cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf"}, - {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517"}, - {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735"}, - {file = "cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e"}, - {file = "cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a"}, - {file = "cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80"}, - {file = "cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be"}, -] - [[package]] name = "cfgv" version = "3.5.0" @@ -413,46 +387,6 @@ files = [ {file = "croniter-6.2.4.tar.gz", hash = "sha256:fc124f751b1b04805c2a04b061898b436b45ab2320b045e1e052ea895de65189"}, ] -[[package]] -name = "cryptography" -version = "50.0.0" -requires_python = "!=3.9.0,!=3.9.1,>=3.9" -summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -groups = ["default"] -dependencies = [ - "cffi>=2.0.0; platform_python_implementation != \"PyPy\"", - "typing-extensions>=4.13.2; python_full_version < \"3.11\"", -] -files = [ - {file = "cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03"}, - {file = "cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645"}, - {file = "cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7"}, - {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3"}, - {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f"}, - {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae"}, - {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a"}, - {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987"}, - {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169"}, - {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f"}, - {file = "cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105"}, - {file = "cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef"}, - {file = "cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30"}, - {file = "cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07"}, - {file = "cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3"}, - {file = "cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f"}, - {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5"}, - {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f"}, - {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025"}, - {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a"}, - {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b"}, - {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708"}, - {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47"}, - {file = "cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9"}, - {file = "cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7"}, - {file = "cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba"}, - {file = "cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9"}, -] - [[package]] name = "cssselect" version = "1.5.0" @@ -1350,18 +1284,6 @@ files = [ {file = "puremagic-2.2.0.tar.gz", hash = "sha256:eb4bddf07c177c4b434554b92165b67449f5a51e152b976202d6254498810eef"}, ] -[[package]] -name = "pycparser" -version = "3.0" -requires_python = ">=3.10" -summary = "C parser in Python" -groups = ["default"] -marker = "implementation_name != \"PyPy\" and platform_python_implementation != \"PyPy\"" -files = [ - {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, - {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, -] - [[package]] name = "pydantic" version = "2.13.4" @@ -1423,24 +1345,6 @@ files = [ {file = "pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117"}, ] -[[package]] -name = "pygithub" -version = "2.9.1" -requires_python = ">=3.9" -summary = "Use the full Github API v3" -groups = ["default"] -dependencies = [ - "pyjwt[crypto]>=2.4.0", - "pynacl>=1.4.0", - "requests>=2.14.0", - "typing-extensions>=4.5.0", - "urllib3>=1.26.0", -] -files = [ - {file = "pygithub-2.9.1-py3-none-any.whl", hash = "sha256:2ec78fca30092d51a42d76f4ddb02131b6f0c666a35dfdf364cf302cdda115b9"}, - {file = "pygithub-2.9.1.tar.gz", hash = "sha256:59771d7ff63d54d427be2e7d0dad2208dfffc2b0a045fec959263787739b611c"}, -] - [[package]] name = "pygments" version = "2.21.0" @@ -1475,48 +1379,6 @@ files = [ {file = "pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423"}, ] -[[package]] -name = "pyjwt" -version = "2.13.0" -extras = ["crypto"] -requires_python = ">=3.9" -summary = "JSON Web Token implementation in Python" -groups = ["default"] -dependencies = [ - "cryptography>=3.4.0", - "pyjwt==2.13.0", -] -files = [ - {file = "pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728"}, - {file = "pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423"}, -] - -[[package]] -name = "pynacl" -version = "1.6.2" -requires_python = ">=3.8" -summary = "Python binding to the Networking and Cryptography (NaCl) library" -groups = ["default"] -dependencies = [ - "cffi>=1.4.1; platform_python_implementation != \"PyPy\" and python_version < \"3.9\"", - "cffi>=2.0.0; platform_python_implementation != \"PyPy\" and python_version >= \"3.9\"", -] -files = [ - {file = "pynacl-1.6.2-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465"}, - {file = "pynacl-1.6.2-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0"}, - {file = "pynacl-1.6.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4"}, - {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87"}, - {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c"}, - {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130"}, - {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6"}, - {file = "pynacl-1.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e"}, - {file = "pynacl-1.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577"}, - {file = "pynacl-1.6.2-cp38-abi3-win32.whl", hash = "sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa"}, - {file = "pynacl-1.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0"}, - {file = "pynacl-1.6.2-cp38-abi3-win_arm64.whl", hash = "sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c"}, - {file = "pynacl-1.6.2.tar.gz", hash = "sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c"}, -] - [[package]] name = "pyotp" version = "2.10.0" diff --git a/pyproject.toml b/pyproject.toml index d9ad111..f42094c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,6 @@ dependencies = [ "puremagic>=2.2.0,<3.0.0", "feedparser>=6.0.12,<7.0.0", "trafilatura>=2.1.0,<3.0.0", - "PyGithub>=2.8.1,<3.0.0", "twikit>=2.3.3,<3.0.0", "jsonschema<5,>=4.25", "croniter<7,>=6",