Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ Reason in English. Communicate with humans in Chinese. Call the user “Sir”.
- Expensive core-py internal design: `docs/30-unit-tdd/`
- Runtime, packaging, migration, observability, and recovery: `docs/40-deployment/`
- Vulnerability reporting: `SECURITY.md`; local security boundaries: `docs/30-unit-tdd/security-model.md`
- Volatile task control: `tasks/`; never treat it as durable truth
- Volatile task control: `tasks/`; never treat it as durable truth, but retain an active packet until its parent task closes
- Mechanically enforceable facts: code, configuration, schemas, tests, assertions, lint, and CI
- Repeated subtree hazards only: the nearest local `AGENTS.md`

Expand All @@ -24,9 +24,12 @@ Resolve the semantic owner before adding durable material. A Unit is a logical r
and [contribution workflow](https://github.com/InKCre/.github/blob/main/CONTRIBUTING.md) for branches, pull requests,
release authority, and delivery boundaries; repository-local documents own exact commands.
- Before a reference-sensitive, logic-altering, or non-obviously-local durable mutation, state the exact object, `From -> To`, side effects, blast radius, invariants, verification, and uncertainty.
- Before promoting behavior, evaluate delivery owner, durable owner, interface layer, and external capability owner independently. Importance, first-party distribution, current pressure, or successful acceptance on one axis does not prove another.
- Before owning external protocol mechanics, inspect existing dependencies and primary documentation and name the unsupported gap. Keep only the application-specific remainder.
- Read the nearest local `AGENTS.md` before changing its subtree. Read shared Product or Product TDD only when that owner is implicated, then the relevant local Unit TDD or Deployment document.
- Before a security-sensitive claim, read the security model and name actor, capability, asset, boundary, harm, and attack path. Missing defense in depth is hardening unless evidence shows a boundary violation.
- Exclude `tasks/`, generated output, dependencies, environments, caches, and temporary directories from ordinary source and durable-doc search unless they are the evidence target.
- Clean task artifacts by parent-task lifecycle, not directory class, age, size, or completed child units. Splitting content must not create a second control authority.
- Use sub-agents only when bounded isolation or parallel capacity repays assignment, validation, integration, conflict, and residual cost. Primary owns the Human relationship, global integration, and material residual.

Pause for Human input when the requested change conflicts with Product/Technical truth, ownership across durable surfaces remains unclear, evidence cannot support a bug or architecture decision, or the shortcut would damage maintainability.
Expand Down
28 changes: 6 additions & 22 deletions app/business/source/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from typing import Optional as Opt

from app.engine import SessionLocal
from app.database_contract.profile import BUILTIN_SOURCE_TYPES_BY_ID
from app.business.info_base.block import BlockManager
from app.schemas.info_base.block import BlockForm, BlockModel
from app.schemas.job import JobModel
Expand Down Expand Up @@ -168,30 +167,15 @@ def sync_source_types(
registered = cls._SOURCE_CLASSES if source_classes is None else source_classes
with SessionLocal() as db:
for source_type, source_cls in registered.items():
builtin = BUILTIN_SOURCE_TYPES_BY_ID.get(source_type)
stmt = sqlalchemy.dialects.postgresql.insert(SourceTypesModel).values(
id=source_type,
description=(
builtin.description
if builtin is not None
else source_cls.__doc__ or "No description."
),
config_schema=(
builtin.config_schema if builtin is not None else source_cls.__configschema__
),
collect_config_schema=(
builtin.collect_config_schema
if builtin is not None
else source_cls.__collectconfigcls__.model_json_schema()
),
description=source_cls.__doc__ or "No description.",
config_schema=source_cls.__configschema__,
collect_config_schema=source_cls.__collectconfigcls__.model_json_schema(),
backfill_config_schema=(
builtin.backfill_config_schema
if builtin is not None
else (
None
if source_cls.__backfillconfigcls__ is None
else source_cls.__backfillconfigcls__.model_json_schema()
)
None
if source_cls.__backfillconfigcls__ is None
else source_cls.__backfillconfigcls__.model_json_schema()
),
)
stmt = stmt.on_conflict_do_update(
Expand Down
30 changes: 0 additions & 30 deletions app/database_contract/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from .profile import (
BUILTIN_AI_DIALECTS,
BUILTIN_JOB_TYPES,
BUILTIN_SOURCE_TYPES,
BUILTIN_STORAGES,
BUILTIN_STORAGE_TYPES,
)
Expand Down Expand Up @@ -119,35 +118,6 @@ def reconcile_builtins(database_url: str | None = None) -> None:
),
)

for source_type in BUILTIN_SOURCE_TYPES:
cursor.execute(
sql.SQL(
"""
INSERT INTO {}.sources_types (
id, description, config_schema, collect_config_schema,
backfill_config_schema
)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (id) DO UPDATE
SET description = EXCLUDED.description,
config_schema = EXCLUDED.config_schema,
collect_config_schema = EXCLUDED.collect_config_schema,
backfill_config_schema = EXCLUDED.backfill_config_schema
"""
).format(sql.Identifier(PROTOCOL_SCHEMA)),
(
source_type.id,
source_type.description,
Jsonb(source_type.config_schema),
Jsonb(source_type.collect_config_schema),
(
None
if source_type.backfill_config_schema is None
else Jsonb(source_type.backfill_config_schema)
),
),
)

for storage in BUILTIN_STORAGES:
cursor.execute(
sql.SQL(
Expand Down
231 changes: 0 additions & 231 deletions app/database_contract/profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,6 @@ class StorageTypeProfile:
writable: bool


@dataclass(frozen=True)
class SourceTypeProfile:
id: str
description: str
config_schema: JsonObject
collect_config_schema: JsonObject
backfill_config_schema: JsonObject | None = None


@dataclass(frozen=True)
class JobTypeProfile:
id: str
Expand Down Expand Up @@ -64,14 +55,6 @@ def _object_schema(
}


def _string(default: str = "") -> JsonObject:
return {"default": default, "type": "string"}


def _integer(default: int) -> JsonObject:
return {"default": default, "type": "integer"}


def _positive_integer(default: int) -> JsonObject:
return {"default": default, "exclusiveMinimum": 0, "type": "integer"}

Expand Down Expand Up @@ -308,225 +291,11 @@ def _boolean(default: bool) -> JsonObject:
)


EMPTY_SOURCE_COMMAND_SCHEMA = {
"additionalProperties": False,
"properties": {},
"title": "EmptySourceCommandConfig",
"type": "object",
}

GITHUB_COLLECT_SCHEMA = {
"additionalProperties": False,
"properties": {"full": _boolean(False)},
"title": "CollectConfig",
"type": "object",
}

TWITTER_COLLECT_SCHEMA = {
"additionalProperties": False,
"properties": {
"full": _boolean(False),
"result_limit": {"default": 40, "maximum": 100, "minimum": 5, "type": "integer"},
},
"title": "CollectConfig",
"type": "object",
}

FEED_COLLECT_SCHEMA = {
"additionalProperties": False,
"description": "Per-run overrides admitted by the RSS collection command.",
"properties": {
"download_enclosures": {
"anyOf": [{"type": "boolean"}, {"type": "null"}],
"default": None,
"title": "Download Enclosures",
},
"fetch_full_text": {
"anyOf": [{"type": "boolean"}, {"type": "null"}],
"default": None,
"title": "Fetch Full Text",
},
"target_storage_id": {
"anyOf": [{"type": "integer"}, {"type": "null"}],
"default": None,
"title": "Target Storage Id",
},
},
"title": "FeedCollectJobConfig",
"type": "object",
}

_FEED_PROPERTIES = {
"feed_url": {"type": "string"},
"request_timeout_seconds": _positive_integer(30),
"max_feed_bytes": _positive_integer(8 * 1024 * 1024),
"fetch_full_text": _boolean(True),
"max_article_bytes": _positive_integer(8 * 1024 * 1024),
"download_enclosures": _boolean(False),
"max_enclosure_bytes": _positive_integer(64 * 1024 * 1024),
"target_storage_id": _integer(-4),
"unidentified_item_behavior": {
"default": "create",
"enum": ["create", "discard"],
"type": "string",
},
"user_agent": _string("InKCre RSS/0.1"),
}


def _feed_source_schema(title: str, description: str) -> JsonObject:
schema = _object_schema(title, description, _FEED_PROPERTIES)
schema["additionalProperties"] = False
schema["required"] = ["feed_url"]
return schema


MAIL_SOURCE_SCHEMA = {
"additionalProperties": False,
"description": "Configuration for one Mail access context.",
"properties": {
"protocol": {"const": "imap", "default": "imap", "type": "string"},
"parameters": {
"additionalProperties": False,
"properties": {
"host": {"minLength": 1, "type": "string"},
"port": {"default": 993, "maximum": 65535, "minimum": 1, "type": "integer"},
"security": {
"default": "tls",
"enum": ["tls", "starttls", "plain"],
"type": "string",
},
"username": {"minLength": 1, "type": "string"},
"password": {"minLength": 1, "type": "string"},
},
"required": ["host", "username", "password"],
"title": "IMAPParameters",
"type": "object",
},
"excluded_mailboxes": {
"anyOf": [
{
"additionalProperties": False,
"properties": {
"names": {
"items": {"type": "string"},
"title": "Names",
"type": "array",
},
"special_uses": {
"items": {"type": "string"},
"title": "Special Uses",
"type": "array",
},
},
"title": "MailboxExclusionPolicy",
"type": "object",
},
{"type": "null"},
],
"default": None,
},
"ordinary_mark_as_seen": _boolean(True),
"backfill_mark_as_seen": _boolean(False),
"synchronize_deletions": _boolean(False),
},
"required": ["parameters"],
"title": "MailSourceConfig",
"type": "object",
}

MAIL_BACKFILL_SCHEMA = {
"additionalProperties": False,
"description": "One exact historical Mail collection range.",
"properties": {
"since": {"format": "date", "title": "Since", "type": "string"},
"before": {
"anyOf": [{"format": "date", "type": "string"}, {"type": "null"}],
"default": None,
"title": "Before",
},
},
"required": ["since"],
"title": "MailBackfillConfig",
"type": "object",
}


BUILTIN_SOURCE_TYPES = (
SourceTypeProfile(
"extensions.github.stars.Source",
"GitHub Stars Source - collects starred repositories from GitHub.",
_object_schema(
"SourceConfig",
"Configuration of GitHub Stars Source.",
{
"github_token": _string(),
"include_private": _boolean(False),
"username": _string(),
},
),
GITHUB_COLLECT_SCHEMA,
),
SourceTypeProfile(
"extensions.mail.source.Source",
"Mail Source - incrementally collects communication records through IMAP.",
MAIL_SOURCE_SCHEMA,
EMPTY_SOURCE_COMMAND_SCHEMA,
MAIL_BACKFILL_SCHEMA,
),
SourceTypeProfile(
"extensions.rss.atom.Source",
"Atom Feed Source.",
_feed_source_schema(
"AtomSourceConfig",
"Configuration for Atom feed source.",
),
FEED_COLLECT_SCHEMA,
),
SourceTypeProfile(
"extensions.rss.rss.Source",
"RSS 2.0 Feed Source.",
_feed_source_schema(
"RssSourceConfig",
"Configuration for RSS 2.0 source.",
),
FEED_COLLECT_SCHEMA,
),
SourceTypeProfile(
"extensions.telegram.source.Source",
"Telegram Source - collects messages sent to the configured Telegram bot.",
_object_schema(
"SourceConfig",
"Configuration of Telegram Source.",
{
"bot_token": _string(),
"collect_method": {
"default": "default",
"enum": ["default", "webhook"],
"type": "string",
},
},
),
EMPTY_SOURCE_COMMAND_SCHEMA,
),
SourceTypeProfile(
"extensions.twitter.bookmark.Source",
"Twitter Bookmark as Source",
_object_schema(
"SourceConfig",
"Configuration for Twitter Bookmark Source.",
{},
),
TWITTER_COLLECT_SCHEMA,
),
)

BUILTIN_STORAGES = (
StorageProfile(-1, "http", "HTTP", {}),
StorageProfile(-4, "postgresql_binary", "PostgreSQL Binary", {}),
)

BUILTIN_AI_DIALECTS_BY_ID = {item.id: item for item in BUILTIN_AI_DIALECTS}
BUILTIN_STORAGE_TYPES_BY_ID = {item.id: item for item in BUILTIN_STORAGE_TYPES}
BUILTIN_SOURCE_TYPES_BY_ID = {item.id: item for item in BUILTIN_SOURCE_TYPES}
BUILTIN_JOB_TYPES_BY_ID = {item.id: item for item in BUILTIN_JOB_TYPES}
17 changes: 0 additions & 17 deletions app/database_contract/readiness.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
from .profile import (
BUILTIN_AI_DIALECTS,
BUILTIN_JOB_TYPES,
BUILTIN_SOURCE_TYPES,
BUILTIN_STORAGES,
BUILTIN_STORAGE_TYPES,
)
Expand Down Expand Up @@ -484,22 +483,6 @@ def _catalog_component(cursor) -> dict[str, Any]:
):
problems.append(f"job_types:{profile.id}")

for profile in BUILTIN_SOURCE_TYPES:
cursor.execute(
sql.SQL(
"SELECT description, config_schema, collect_config_schema, "
"backfill_config_schema FROM {}.sources_types WHERE id = %s"
).format(sql.Identifier(PROTOCOL_SCHEMA)),
(profile.id,),
)
if cursor.fetchone() != (
profile.description,
profile.config_schema,
profile.collect_config_schema,
profile.backfill_config_schema,
):
problems.append(f"sources_types:{profile.id}")

for storage in BUILTIN_STORAGES:
cursor.execute(
sql.SQL("SELECT type, nickname, config FROM {}.storages WHERE id = %s").format(
Expand Down
Loading
Loading