diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b90d55c9f..f84a3b139 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,59 @@ jobs: - name: Integration tests run: make integration + integration-milvus: + name: integration tests (Milvus 2.6) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.2.0 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Set up Python + run: uv python install 3.12 + + - name: Install dependencies (frozen) + run: make install-deps + + - name: Start Milvus 2.6.22 + run: | + curl --fail --location --retry 3 \ + --output /tmp/milvus-compose.yml \ + https://github.com/milvus-io/milvus/releases/download/v2.6.22/milvus-standalone-docker-compose.yml + docker compose -f /tmp/milvus-compose.yml up -d + for attempt in {1..60}; do + if curl --fail --silent http://127.0.0.1:9091/healthz >/dev/null; then + exit 0 + fi + sleep 2 + done + docker compose -f /tmp/milvus-compose.yml logs + exit 1 + + - name: Milvus repository contract + env: + EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 + EVEROS_TEST_MILVUS_FULL_STARTUP: "1" + run: uv run --frozen pytest tests/integration/test_milvus_remote.py -v + + # The contract tests above exercise the repository port directly. This + # drives the same request path the application does -- /add, /flush, + # cascade, every search method, /get -- against the remote backend, so a + # break anywhere between the API and Milvus surfaces here rather than in + # production. Deselecting the LanceDB half keeps the job to its subject. + - name: Milvus end-to-end (tiered API suites) + env: + EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 + run: uv run --frozen pytest tests/integration/test_tiers -v -k milvus + + - name: Stop Milvus + if: always() + run: docker compose -f /tmp/milvus-compose.yml down -v + package: name: package build runs-on: ubuntu-latest diff --git a/config.example.toml b/config.example.toml index f8043127f..48cff25fd 100644 --- a/config.example.toml +++ b/config.example.toml @@ -56,3 +56,15 @@ max_concurrent = 5 # # [lancedb] # read_consistency_seconds = 5.0 +# +# [index] +# backend = "lancedb" # or "milvus" +# +# # Required only when index.backend = "milvus". Install with: +# # pip install "everos[milvus]" +# [milvus] +# uri = "http://127.0.0.1:19530" # Milvus Server or Zilliz Cloud endpoint +# token = "" # e.g. "root:Milvus" or cloud token +# db_name = "" +# consistency_level = "Session" +# collection_prefix = "everos" diff --git a/docs/api.md b/docs/api.md index 422b47ffa..d19361f4e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -93,8 +93,9 @@ bare FastAPI `detail`); see [Errors](#errors). `/add` and `/flush` write the markdown file (the source of truth) **synchronously** — when the call returns with `status: "extracted"`, -the new entry exists on disk. The LanceDB vector / BM25 / scalar index -is rebuilt by the in-process **cascade coroutine asynchronously**. +the new entry exists on disk. The configured vector / BM25 / scalar +index backend is rebuilt by the in-process **cascade coroutine +asynchronously**. That means `/search` and `/get` may not see a record immediately after the `/flush` that produced it. Typical sync latency is sub-second, but @@ -371,7 +372,7 @@ A recursive boolean tree of predicates. Used by `/search.filters` and `/get.filters`. The Pydantic envelope only checks the recursive combinator shape; field-level validity (which scalar fields are filterable, which operators apply, value coercion) runs when the -node is compiled to a LanceDB `where` clause server-side. Compile +node is compiled to a backend-specific filter clause server-side. Compile errors surface as `422` with the offending field / operator in `error.message`. @@ -464,12 +465,12 @@ Examples: |---|---| | `"keyword"` | BM25 only — pure lexical match, no embedding cost | | `"vector"` | Dense vector ANN only — semantic recall, no lexical | -| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter in a single LanceDB query | +| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter against the configured derived index backend | | `"agentic"` | Iterative cluster-path retrieval driven by a cross-encoder rerank loop; higher quality at higher latency / cost | `"hybrid"` is the default because it balances recall and precision -with one LanceDB roundtrip. `"agentic"` calls the LLM in a loop and -should be reserved for offline or background workflows. +without requiring the agentic loop. `"agentic"` calls the LLM in a loop +and should be reserved for offline or background workflows. ### GetMemoryType @@ -613,7 +614,7 @@ scope. this `(session_id, app_id, project_id)`, or it was already flushed). `/flush` is synchronous with respect to markdown persistence: by the -time the response returns, the new entry is on disk. LanceDB index +time the response returns, the new entry is on disk. Derived index sync is still asynchronous — see [Eventual consistency](#eventual-consistency). diff --git a/docs/architecture.md b/docs/architecture.md index afb5efe30..4aae7826a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,7 +17,7 @@ │ + reflection + strategies + get + events │ ├──────────────────────────────────────────────────────┤ │ infra/persistence (Storage adapters; infra/ may host other adapter types) │ -│ markdown + sqlite + lancedb │ +│ markdown + sqlite + derived index │ └──────────────────────────────────────────────────────┘ Cross-cutting (used by all layers, depends on none): @@ -68,8 +68,8 @@ layers = [ └────────────────────────────────────────────────────────────────┘ ┌──────────────┐ ┌──────────────┐ ┌─────────────────┐ - │ Markdown │ │ SQLite │ │ LanceDB │ - │ (truth) │ │ (state) │ │ (index) │ + │ Markdown │ │ SQLite │ │ Derived index │ + │ (truth) │ │ (state) │ │ LanceDB/Milvus │ ├──────────────┤ ├──────────────┤ ├─────────────────┤ │ entries + │ │ change queue │ │ vector ANN │ │ frontmatter │ │ + state/LSN │ │ BM25 (Tantivy) │ @@ -78,7 +78,7 @@ layers = [ └──────────────┘ └──────────────┘ └─────────────────┘ │ │ │ ▼ ▼ ▼ - memory-root/ .index/sqlite/ .index/lancedb/ + memory-root/ .index/sqlite/ .index// (truth source) (system data) (rebuildable) ``` @@ -101,10 +101,13 @@ External message │ │ ▼ ▼ 4a. SQLite 4b. memory.cascade (async daemon) - audit watches md → diff entries → LanceDB sync + audit watches md → diff entries → index sync ``` -**Key guarantee**: md write is strongly consistent (fsync). LanceDB is eventually consistent. LanceDB unavailability does not block response — changes buffer in the SQLite `md_change_state` queue, replayed on recovery. +**Key guarantee**: md write is strongly consistent (fsync). The derived +index is eventually consistent. Index backend unavailability does not block +response — changes buffer in the SQLite `md_change_state` queue, replayed on +recovery. ## Read path @@ -115,8 +118,8 @@ User query 1. service.search │ ▼ -2. memory.search (hybrid) single LanceDB query = - BM25 + vector ANN + scalar filter +2. memory.search (hybrid) BM25 + vector ANN + scalar filter + through the configured index backend │ ▼ 3. (optional) read md original markdown for context @@ -139,12 +142,13 @@ extract/ ### `memory/cascade/` -Daemon that watches markdown changes and syncs to LanceDB: +Daemon that watches markdown changes and syncs to the configured derived +index backend: - inotify / FSEvents file watcher (cross-platform via `watchdog`) - 500ms debounce - Entry-level diff (added / changed / removed) -- LanceDB single-transaction update (text + vector columns atomic) +- Per-entry index upsert / delete (text + vector columns update together) - LSN-based crash recovery via the SQLite `md_change_state` queue - Handlers for all eight business kinds: episode, atomic_fact, foresight, user_profile, agent_case, agent_skill, knowledge_document, knowledge_topic @@ -225,7 +229,7 @@ holding **only memory extraction algorithms**: everalgo is: - **Stateless** — pure functions, no class hierarchy -- **No I/O** — does not touch md files / LanceDB / SQLite +- **No I/O** — does not touch md files, derived indexes, or SQLite - **No prompts inline** — extractors that accept a prompt-override parameter use the project-supplied value; others use their algo-bundled defaults This boundary lets everalgo be reused across product forms (this open-source build, EverOS Cloud, OpenClaw plugins, etc.). diff --git a/docs/cascade_runbook.md b/docs/cascade_runbook.md index 3e0dcd900..338d2a7cf 100644 --- a/docs/cascade_runbook.md +++ b/docs/cascade_runbook.md @@ -1,9 +1,13 @@ # Cascade Runbook -The cascade daemon keeps LanceDB in sync with the markdown files under -the memory root. Service / entry points only ever write markdown; the -daemon is the **sole** writer of the LanceDB index. This runbook covers -the recurring operational questions. +The cascade daemon keeps the configured derived index in sync with the +markdown files under the memory root. Service / entry points only ever write +markdown; the daemon is the **sole** writer of the derived index. This runbook +covers the recurring operational questions. + +Sections that mention LanceDB-specific schemas, index cache, file descriptors, +or `lance error` messages apply to the default LanceDB backend. Milvus uses the +same cascade queue with backend-specific collection management. ## What runs where @@ -13,7 +17,7 @@ providers in order: 1. **Metrics** — Prometheus collector. 2. **LLM** — LLM client initialisation. 3. **SQLite** — system DB + schema (`SQLModel.metadata.create_all`). -4. **LanceDB** — async connection + schema verification + FTS indexes. +4. **Derived index** — async connection + schema verification + search indexes. 5. **Cascade** — watcher + scanner + worker, all in-process tasks. 6. **OME** — offline memory engine. @@ -23,7 +27,7 @@ The cascade subsystem itself is three independent loops: |---|---|---| | Watcher | `watchdog` filesystem events (sync thread) | `md_change_state.upsert` per registered kind | | Scanner | Periodic walk (`scan_interval_seconds`, default 30 s) | Same — catches changes the watcher missed | -| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → LanceDB upsert / delete | +| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → index upsert / delete | Every loop talks to the same `md_change_state` sqlite table. The worker's claim mode (`pending → processing → done/failed`) keeps @@ -123,7 +127,7 @@ parallel with a live `everos server`. ## Rebuild the index: `everos cascade rebuild` -The safe recovery from a drifted or corrupt LanceDB index. It rebuilds +The safe recovery from a drifted or corrupt derived index. It rebuilds the whole index from markdown (the source of truth) in one shot: ```bash @@ -132,15 +136,16 @@ everos cascade rebuild --yes # non-interactive ``` > **Stop the `everos server` first.** Unlike `cascade sync`, rebuild -> **drops and recreates** the LanceDB tables. A running daemon holds +> **drops and recreates** the active backend's tables or collections. A running +> daemon holds > cached table handles that would keep pointing at (and writing to) the > dropped dataset, corrupting the rebuild. This is the one cascade > command that is **not** safe to run alongside a live server. What it does, in order: -1. **Drops** every business LanceDB table (`drop_business_tables`) and - evicts them from the connection cache. +1. **Drops** every business table or collection (`drop_business_tables`) and + evicts it from the process cache. 2. **Recreates** them empty from the current schema + FTS indexes (`ensure_business_indexes`). 3. **Clears** the cascade queue (`md_change_state.reset_all`) so every @@ -160,6 +165,10 @@ Why not a bare `rm`: | `rm -rf .index` | ✅ | ❌ deletes un-extracted messages | | `everos cascade rebuild` | ✅ | ✅ | +For a remote Milvus backend, rebuild acts only on collections whose names use +the configured `collection_prefix`. Use a unique prefix or dedicated database +before running it on a shared Milvus Server or Zilliz Cloud deployment. + ## Recovery paths ### LanceDB schema drift on startup diff --git a/docs/configuration.md b/docs/configuration.md index 98fdf3479..e47fbc80e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,6 +99,29 @@ everos init --root /data/everos | `read_consistency_seconds` | float \| null | `null` | Read consistency interval. `null` = no check, `0` = strict, `>0` = eventual. | | `index_cache_size_bytes` | int | `16777216` | Upper bound on LanceDB index cache (16 MB default). | +### `[index]` + +The derived vector/BM25 index is rebuildable; Markdown remains the source of +truth regardless of backend. + +| Field | Type | Default | Description | +|---|---|---|---| +| `backend` | string | `"lancedb"` | Index implementation: `lancedb` or `milvus`. | + +### `[milvus]` + +Used only when `index.backend = "milvus"`. Install the optional client with +`pip install "everos[milvus]"`. EverOS supports a remote Milvus Server or +Zilliz Cloud endpoint; a Milvus Lite filesystem path is rejected. + +| Field | Type | Default | Description | +|---|---|---|---| +| `uri` | string | `""` | Milvus Server or Zilliz Cloud URI. | +| `token` | string | `""` | Authentication token, when required. | +| `db_name` | string | `""` | Optional Milvus database name. | +| `consistency_level` | string | `"Session"` | Milvus consistency level used by collections. | +| `collection_prefix` | string | `"everos"` | Prefix for the seven derived-index collections. | + ### `[llm]` | Field | Type | Default | Required | Description | diff --git a/docs/knowledge.md b/docs/knowledge.md index d3ad92dd3..a7d7b1770 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -48,8 +48,8 @@ Each level corresponds to a different granularity of API: ## Storage layout Every document is a self-contained directory. Markdown files are the -single source of truth; SQLite and LanceDB are derived indexes built -automatically by the cascade daemon. +single source of truth; SQLite and the configured vector/BM25 index are +derived indexes built automatically by the cascade daemon. ``` ~/.everos///knowledge/ @@ -73,16 +73,16 @@ automatically by the cascade daemon. ### Storage roles ``` -Markdown (source of truth) + SQLite (structured state) + LanceDB (vector + BM25 index) +Markdown (source of truth) + SQLite (structured state) + derived vector/BM25 index ``` | Store | What it holds | Role | |-------|---------------|------| | Markdown | Document metadata, summaries, topic content, original files | Single source of truth; human-readable and editable | | SQLite | Document rows, topic rows (with content), change queue | Structured queries, paginated lists, count aggregation | -| LanceDB | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | +| Derived index | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | -Even if SQLite and LanceDB data is corrupted, as long as the Markdown +Even if SQLite and derived index data is corrupted, as long as the Markdown files are intact, the indexes can be fully rebuilt via the cascade daemon. ### Markdown format @@ -500,7 +500,7 @@ query ─→ embed ─→ keyword (BM25) ─┐ ``` 1. **Embed** — the query is embedded using the configured embedding provider -2. **Recall** — dual-channel retrieval from LanceDB: +2. **Recall** — dual-channel retrieval from the configured derived index: - BM25 channel: keyword matching on `summary_tokens` + `content_tokens` - ANN channel: nearest-neighbor search on the `vector` column - In `hybrid` mode, both channels run in parallel @@ -531,7 +531,7 @@ export EVEROS_KNOWLEDGE__SEARCH__RERANK_N=100 ## Cascade sync The cascade daemon watches the knowledge Markdown directory for file -changes and keeps SQLite + LanceDB in sync. +changes and keeps SQLite + the configured derived index in sync. ``` md file written @@ -539,7 +539,7 @@ md file written → worker picks up from queue (≤1s poll interval) → handler dispatched by file type: index.md → KnowledgeDocumentHandler → SQLite upsert (metadata) - N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + LanceDB upsert + N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + index upsert ``` The topic handler uses a SHA-256 content digest to skip unchanged files — @@ -597,7 +597,7 @@ curl .../documents?app_id=tenant_b&project_id=proj_1 # → { "documents": [], "total": 0 } ``` -Storage paths, SQLite rows, and LanceDB indexes are all scoped by +Storage paths, SQLite rows, and derived index rows are all scoped by `app_id` + `project_id`. ## End-to-end walkthrough diff --git a/docs/multimodal.md b/docs/multimodal.md index eeb181811..5c6b1a6da 100644 --- a/docs/multimodal.md +++ b/docs/multimodal.md @@ -41,7 +41,7 @@ POST /api/v2/memory/add boundary detector → extraction LLM → memory cell (MemCell) │ ▼ - markdown (truth) + SQLite (state) + LanceDB (vector + BM25) + markdown (truth) + SQLite (state) + derived vector/BM25 index │ ▼ retrievable via /search and /get like any text memory diff --git a/docs/overview.md b/docs/overview.md index 07297673d..0bfda5b7d 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -11,7 +11,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory - Local deployment for personal agents or small teams - Conversation, workflow, agent-trace, file-knowledge → structured memory - Hybrid retrieval (BM25 + vector + scalar filter) -- Cascade index sync (md edit → LanceDB sub-second) +- Cascade index sync (md edit → derived index) - Dual-track memory (user-track / agent-track) - Offline memory evolution (Foresight / AtomicFact / Profile / Skill), including Reflection — a consolidation strategy within the OME that @@ -30,7 +30,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory ### 1. Markdown as Source of Truth ``` -delete all LanceDB / SQLite files → can rebuild from md +delete all derived index / SQLite files → can rebuild from md delete any md file → memory is gone ``` @@ -42,7 +42,7 @@ User trust comes from physical visibility — the user can `cat` / `vim` / `grep |---|---|---| | Markdown files | Truth source — entries, frontmatter | Search (grep is degraded fallback only) | | SQLite | Queue, cascade audit log, sensitive data isolation | Vector / full-text | -| LanceDB | Vector ANN + BM25 + scalar filter, single-query hybrid | Be the source of truth (loss = rebuild from md) | +| Derived index | Vector ANN + BM25 + scalar filter | Be the source of truth (loss = rebuild from md) | ### 3. Algorithm-orchestration separation diff --git a/docs/reflection.md b/docs/reflection.md index 63c09f1fd..f3e920bcc 100644 --- a/docs/reflection.md +++ b/docs/reflection.md @@ -147,14 +147,15 @@ narrative. ## Storage layout -Memory uses Markdown as the single source of truth; SQLite and LanceDB are -derived indexes built automatically by the cascade daemon. +Memory uses Markdown as the single source of truth; SQLite and the configured +vector/BM25 index backend are derived indexes built automatically by the +cascade daemon. | Store | What it holds | Role | |---|---|---| | Markdown | Episode bodies, merged narratives, archive markers | Single source of truth; human-readable and editable | | SQLite | Clusters and members, consolidation audit records | Structured state and queries | -| LanceDB | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | +| Derived index | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | The **merged narrative** is written to the Episode daily-log Markdown; its frontmatter marks that it came from a cluster: @@ -186,7 +187,7 @@ deprecated_entries: --- ``` -> Soft-archive, not delete: even if SQLite / LanceDB are corrupted, as long +> Soft-archive, not delete: even if SQLite / the derived index are corrupted, as long > as the Markdown is intact the indexes can be fully rebuilt — and every > consolidation remains traceable back to its original content. @@ -355,5 +356,5 @@ curl -s -X POST "$BASE/memory/search" \ ## See also - [how-memory-works.md](how-memory-works.md) — Episodes and the memory extraction pipeline -- [storage_layout.md](storage_layout.md) — Markdown + SQLite + LanceDB stack +- [storage_layout.md](storage_layout.md) — Markdown + SQLite + derived index stack - [api.md](api.md) — full HTTP API reference diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 860446648..3d90c775d 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -3,8 +3,9 @@ How `everos` lays out a memory-root on disk: directory tree, file naming, frontmatter chassis, and entry-id encoding. -The contents are the **source of truth**; SQLite and LanceDB are -derived indexes that can be rebuilt from markdown alone. +The contents are the **source of truth**; SQLite and the configured +vector/BM25 index backend are derived indexes that can be rebuilt from +markdown alone. ## 1. Memory-root tree @@ -49,8 +50,8 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). │ │ ├── ome.db Offline Memory Engine state │ │ ├── ome.aps.db APScheduler jobstore (split to avoid lock contention) │ │ └── ome.db.lock OME single-engine guard (portalocker) -│ └── lancedb/ -│ └── .lance/ one directory per LanceDB table +│ ├── lancedb/ +│ │ └── .lance/ default derived index backend │ ├── ome.toml user-editable OME strategy overrides (hot-reloaded) └── .tmp/ staging dir for batch / multi-step writes @@ -168,9 +169,9 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers > **File-level seq, not global**: the same `ep_20260601_00000001` may > appear across two different `user_id`s (each user has its own daily file). > Cross-table joins must therefore key on **`(scope_id, entry_id)`** -> rather than `entry_id` alone — see SQLite/LanceDB tables that follow. +> rather than `entry_id` alone — see the derived index tables that follow. -## 5. SQLite + LanceDB derived indexes +## 5. SQLite + derived indexes ``` .index/ @@ -178,9 +179,8 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers │ └── system.db state / audit / cascade queue + buffer / LSN │ (system tables: md_change_state, memcell, │ unprocessed_buffer, conversation_status, cluster) -└── lancedb/ - └── .lance/ one Arrow table per business kind — the per-kind - rows (text / vector / tokens / metadata) live here +├── lancedb/ +│ └── .lance/ default derived index backend ``` - **SQLite** ([`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/)) @@ -190,13 +190,14 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers per-kind business rows. `reflection_report` is the audit trail for Reflection merges (cluster_id, mode, source_members, merged_entry_id, status). -- **LanceDB** ([`infra/persistence/lancedb/tables/`](../src/everos/infra/persistence/lancedb/tables/)) - holds the per-kind business rows, keyed `_` (so - cross-table joins use `(owner_id, entry_id)`); each table's `Vector(N)` - dimension matches the embedding model output. - -Episode and AtomicFact LanceDB tables carry a `deprecated_by: str | None` -column. When an episode is superseded by a Reflection merge, +- The **derived index backend** holds the per-kind business rows, keyed + `_` (so cross-table joins use `(owner_id, entry_id)`). + LanceDB is the default backend under `.index/lancedb/`; Milvus can be enabled + as the same rebuildable index backend and lives outside the memory root in a + configured Milvus Server or Zilliz Cloud deployment. + +Episode and AtomicFact index rows carry a `deprecated_by: str | None` column. +When an episode is superseded by a Reflection merge, `deprecated_by` is set to the merged episode's entry_id. Search filters automatically exclude rows where `deprecated_by IS NOT NULL`. @@ -226,5 +227,5 @@ this primitive is **schema-agnostic** — field-level semantics - Code: - [`core/persistence/memory_root.py`](../src/everos/core/persistence/memory_root.py) - [`core/persistence/markdown/`](../src/everos/core/persistence/markdown/) - - [`infra/persistence/{markdown,sqlite,lancedb}/`](../src/everos/infra/persistence/) - - [`memory/cascade/`](../src/everos/memory/cascade/) (md → LanceDB sync) + - [`infra/persistence/{markdown,sqlite,lancedb,milvus,index}/`](../src/everos/infra/persistence/) + - [`memory/cascade/`](../src/everos/memory/cascade/) (md → derived index sync) diff --git a/pyproject.toml b/pyproject.toml index 991d9c91b..26ecf6984 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ otel = [ "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-http>=1.27.0", ] +milvus = ["pymilvus>=3.0.0"] [project.urls] Homepage = "https://evermind.ai" @@ -226,6 +227,8 @@ forbidden_modules = [ "everos.infra.persistence.markdown.**", "everos.infra.persistence.lancedb.**", "everos.infra.persistence.sqlite.**", + "everos.infra.persistence.index.**", + "everos.infra.persistence.milvus.**", ] # `forbidden` contracts run a *transitive closure* — any path from a # source module to a forbidden module fails the check, even when the @@ -252,6 +255,12 @@ ignore_imports = [ "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.mds", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.writers", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.readers", + "everos.infra.persistence.index -> everos.infra.persistence.index.lancedb", + "everos.infra.persistence.index -> everos.infra.persistence.index.milvus", + "everos.infra.persistence.index -> everos.infra.persistence.index.predicate", + "everos.infra.persistence.index -> everos.infra.persistence.index.protocols", + "everos.infra.persistence.index -> everos.infra.persistence.index.router", + "everos.infra.persistence.index -> everos.infra.persistence.index.schema", ] [[tool.importlinter.contracts]] @@ -280,4 +289,5 @@ dev = [ # [otel] stack is always present in the dev / CI environment. "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-http>=1.27.0", + "pymilvus>=3.0.0", ] diff --git a/src/everos/README.md b/src/everos/README.md index 8cc1cef96..2be6caf25 100644 --- a/src/everos/README.md +++ b/src/everos/README.md @@ -10,7 +10,7 @@ everos/ ├── entrypoints/ Presentation: cli + api ├── service/ Application: use case orchestration ├── memory/ Domain: extract + search + cascade + prompt_slots + models -├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb} +├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb, milvus, index} ├── component/ Cross-cutting providers: llm / embedding / config / utils ├── core/ Runtime base: observability / lifespan / context └── config/ Data: Settings + default.toml + prompt_slots templates diff --git a/src/everos/component/utils/datetime.py b/src/everos/component/utils/datetime.py index feccfecf1..063af50da 100644 --- a/src/everos/component/utils/datetime.py +++ b/src/everos/component/utils/datetime.py @@ -183,6 +183,17 @@ def from_timestamp(ts: int | float) -> _dt.datetime: return _dt.datetime.fromtimestamp(seconds, tz=_display_tz()) +def from_timestamp_ms(ts: int) -> _dt.datetime: + """Parse an epoch-milliseconds value into a timezone-aware datetime. + + Exact inverse of :func:`to_timestamp_ms`. Unlike :func:`from_timestamp` + this does **not** guess whether the value is seconds or milliseconds, so + pre-2001 instants (whose ms value falls under the 1e12 heuristic + threshold) round-trip instead of being read as seconds. + """ + return _dt.datetime.fromtimestamp(ts / 1000.0, tz=_display_tz()) + + def from_iso_format(value: _dt.datetime | int | float | str) -> _dt.datetime: """Parse a value into a timezone-aware datetime (strict). diff --git a/src/everos/config/__init__.py b/src/everos/config/__init__.py index adbd1e354..fb3108132 100644 --- a/src/everos/config/__init__.py +++ b/src/everos/config/__init__.py @@ -15,9 +15,11 @@ from .settings import BoundaryDetectionSettings as BoundaryDetectionSettings from .settings import CascadeSettings as CascadeSettings from .settings import EmbeddingSettings as EmbeddingSettings +from .settings import IndexSettings as IndexSettings from .settings import LanceDBSettings as LanceDBSettings from .settings import LLMSettings as LLMSettings from .settings import MemorySettings as MemorySettings +from .settings import MilvusSettings as MilvusSettings from .settings import MultimodalSettings as MultimodalSettings from .settings import RerankSettings as RerankSettings from .settings import Settings as Settings @@ -28,9 +30,11 @@ __all__ = [ "BoundaryDetectionSettings", "EmbeddingSettings", + "IndexSettings", "LLMSettings", "LanceDBSettings", "MemorySettings", + "MilvusSettings", "MultimodalSettings", "RerankSettings", "Settings", diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index e2137f84a..38bf40030 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -50,6 +50,18 @@ cache_size_kb = 2048 # Uncomment to override: # read_consistency_seconds = 5.0 +[index] +# Rebuildable vector/BM25 index. Markdown remains the source of truth. +backend = "lancedb" + +[milvus] +# Remote Milvus Server or Zilliz Cloud only; Milvus Lite paths are unsupported. +uri = "" +token = "" +db_name = "" +consistency_level = "Session" +collection_prefix = "everos" + [llm] # Provider-agnostic OpenAI-protocol client config. Override via env: # EVEROS_LLM__MODEL, EVEROS_LLM__API_KEY, EVEROS_LLM__BASE_URL diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 50da9947d..5ad9a105e 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -23,6 +23,7 @@ from __future__ import annotations import os +import re from functools import cache from pathlib import Path from typing import Literal @@ -375,6 +376,32 @@ class CascadeSettings(BaseModel): optimize_rebuild_interval_seconds: float = 12 * 60 * 60.0 +class IndexSettings(BaseModel): + """Rebuildable derived-index backend selection.""" + + backend: Literal["lancedb", "milvus"] = "lancedb" + + +class MilvusSettings(BaseModel): + """Remote Milvus Server or Zilliz Cloud connection settings.""" + + uri: str = "" + token: SecretStr = SecretStr("") + db_name: str = "" + consistency_level: Literal["Strong", "Bounded", "Session", "Eventually"] = "Session" + collection_prefix: str = Field(default="everos", min_length=1) + + @field_validator("collection_prefix") + @classmethod + def _validate_collection_prefix(cls, value: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value): + raise ValueError( + "collection_prefix must start with a letter or underscore and " + "contain only letters, digits, and underscores" + ) + return value + + class KnowledgeSearchSettings(BaseModel): """``[knowledge.search]`` — retrieval tuning for the knowledge module.""" @@ -448,6 +475,8 @@ class Settings(BaseSettings): api: ApiSettings = ApiSettings() sqlite: SqliteSettings = SqliteSettings() lancedb: LanceDBSettings = LanceDBSettings() + index: IndexSettings = IndexSettings() + milvus: MilvusSettings = MilvusSettings() llm: LLMSettings = LLMSettings() embedding: EmbeddingSettings = EmbeddingSettings() rerank: RerankSettings = RerankSettings() diff --git a/src/everos/entrypoints/api/lifespans/lancedb.py b/src/everos/entrypoints/api/lifespans/lancedb.py index d20e05b63..8f7a47df2 100644 --- a/src/everos/entrypoints/api/lifespans/lancedb.py +++ b/src/everos/entrypoints/api/lifespans/lancedb.py @@ -31,13 +31,17 @@ from everos.core.lifespan import LifespanProvider from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import ( - BUSINESS_SCHEMAS_WITH_VECTOR, - dispose_connection, - ensure_business_indexes, - get_connection, - get_table, - verify_business_schemas, +from everos.infra.persistence.index import ( + ALL_REPOS, + active_backend, + is_null, + schema_for, +) +from everos.infra.persistence.index import ( + shutdown as shutdown_index, +) +from everos.infra.persistence.index import ( + startup as startup_index, ) logger = get_logger(__name__) @@ -58,14 +62,16 @@ async def _log_unbackfilled_hint() -> None: startup. """ total_null = 0 - for schema in BUSINESS_SCHEMAS_WITH_VECTOR: + for repo in ALL_REPOS: + logical_schema = schema_for(repo.schema) + if not any(field.name == "vector" for field in logical_schema.vector_fields): + continue try: - table = await get_table(schema.TABLE_NAME, schema) - count = await table.count_rows(filter="vector IS NULL") + count = await repo.count_where(is_null("vector")) except Exception as exc: logger.warning( "unbackfilled_check_failed", - schema=schema.__name__, + schema=repo.schema.__name__, error=repr(exc), ) continue @@ -100,12 +106,11 @@ def __init__(self, order: int = 11) -> None: super().__init__(name="lancedb", order=order) async def startup(self, app: FastAPI) -> Any: - conn = await get_connection() - await verify_business_schemas() - await ensure_business_indexes() - await _log_unbackfilled_hint() - logger.info("lancedb_ready", uri=conn.uri) + conn = await startup_index() + if active_backend() == "lancedb": + await _log_unbackfilled_hint() + logger.info("derived_index_ready", backend=active_backend()) return conn async def shutdown(self, app: FastAPI) -> None: - await dispose_connection() + await shutdown_index() diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index 8710f947b..d8095f56e 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -32,6 +32,7 @@ import asyncio import enum import os +from collections.abc import AsyncIterator from contextlib import asynccontextmanager from pathlib import Path from typing import Annotated @@ -46,11 +47,11 @@ from everos.core.persistence import MemoryRoot from everos.entrypoints.cli._log_setup import configure_cli_logging from everos.entrypoints.cli.commands._backfill_cmd import run_backfill -from everos.infra.persistence.lancedb import ( - dispose_connection, +from everos.infra.persistence.index import ( + connect, drop_business_tables, ensure_business_indexes, - get_connection, + shutdown, verify_business_schemas, ) from everos.infra.persistence.sqlite import ( @@ -135,9 +136,7 @@ def _apply_verbose_logging(verbose: bool | None) -> None: @asynccontextmanager -async def _runtime( # type: ignore[no-untyped-def] - *, verify: bool = True, ensure: bool = True -): +async def _runtime(*, verify: bool = True, ensure: bool = True) -> AsyncIterator[None]: """Stand up sqlite + lancedb the same way the API lifespan would. The CLI uses the same lazy, process-wide singletons the API lifespan @@ -163,7 +162,7 @@ async def _runtime( # type: ignore[no-untyped-def] engine = get_engine() async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) - await get_connection() + await connect() if verify: await verify_business_schemas() if ensure: @@ -171,7 +170,7 @@ async def _runtime( # type: ignore[no-untyped-def] try: yield finally: - await dispose_connection() + await shutdown() await dispose_engine() diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py new file mode 100644 index 000000000..419df6b93 --- /dev/null +++ b/src/everos/infra/persistence/index/__init__.py @@ -0,0 +1,204 @@ +"""Backend-neutral facade for the rebuildable BM25/vector index. + +Markdown remains the source of truth and SQLite remains the system-state +store. This boundary owns only the derived business indexes used by cascade, +search, and get. LanceDB and Milvus implement the same typed ports, so callers +do not branch on physical storage. +""" + +from __future__ import annotations + +from typing import Any + +from everos.config import load_settings +from everos.infra.persistence import lancedb as _lancedb + +from .lancedb import ( + agent_case_repo as _lance_agent_case_repo, +) +from .lancedb import ( + agent_skill_repo as _lance_agent_skill_repo, +) +from .lancedb import ( + atomic_fact_repo as _lance_atomic_fact_repo, +) +from .lancedb import ( + episode_repo as _lance_episode_repo, +) +from .lancedb import ( + foresight_repo as _lance_foresight_repo, +) +from .lancedb import ( + knowledge_topic_repo as _lance_knowledge_topic_repo, +) +from .lancedb import ( + lance_index_backend, +) +from .lancedb import ( + user_profile_repo as _lance_user_profile_repo, +) +from .milvus import milvus_index_backend +from .predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, + all_of, + any_of, + compare, + contains, + eq, + gt, + gte, + is_null, + lt, + lte, + ne, + one_of, +) +from .protocols import ( + AgentSkillIndexRepository, + EpisodeIndexRepository, + IndexBackend, + IndexRepository, +) +from .router import ( + RoutedAgentSkillRepository, + RoutedEpisodeRepository, + RoutedIndexRepository, +) +from .schema import IndexField, IndexFieldKind, IndexSchema, schema_for + +AgentCase = _lancedb.AgentCase +AgentSkill = _lancedb.AgentSkill +AtomicFact = _lancedb.AtomicFact +Episode = _lancedb.Episode +Foresight = _lancedb.Foresight +KnowledgeTopic = _lancedb.KnowledgeTopic +ParentType = _lancedb.ParentType +UserProfile = _lancedb.UserProfile + +episode_repo = RoutedEpisodeRepository(_lance_episode_repo, "episode_repo") +atomic_fact_repo = RoutedIndexRepository(_lance_atomic_fact_repo, "atomic_fact_repo") +foresight_repo = RoutedIndexRepository(_lance_foresight_repo, "foresight_repo") +agent_case_repo = RoutedIndexRepository(_lance_agent_case_repo, "agent_case_repo") +agent_skill_repo = RoutedAgentSkillRepository( + _lance_agent_skill_repo, "agent_skill_repo" +) +user_profile_repo = RoutedIndexRepository(_lance_user_profile_repo, "user_profile_repo") +knowledge_topic_repo = RoutedIndexRepository( + _lance_knowledge_topic_repo, "knowledge_topic_repo" +) + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + + +def active_backend() -> str: + """Name of the configured derived-index backend.""" + return load_settings().index.backend + + +def _backend() -> IndexBackend: + if active_backend() == "milvus": + return milvus_index_backend + return lance_index_backend + + +async def connect() -> Any: + return await _backend().connect() + + +async def startup() -> Any: + return await _backend().startup() + + +async def shutdown() -> None: + await _backend().shutdown() + + +async def ensure_business_indexes() -> None: + await _backend().ensure_business_indexes() + + +async def verify_business_schemas() -> None: + await _backend().verify_business_schemas() + + +async def drop_business_tables() -> list[str]: + return await _backend().drop_business_tables() + + +def repo_for_schema(schema: type[Any]) -> IndexRepository[Any]: + """Resolve a registered repository by its logical record model.""" + for repo in ALL_REPOS: + if repo.schema is schema: + return repo + raise KeyError(f"no derived-index repository for {schema!r}") + + +__all__ = [ + "ALL_REPOS", + "AgentCase", + "AgentSkill", + "AgentSkillIndexRepository", + "All", + "AnyOf", + "AtomicFact", + "Comparison", + "Contains", + "Episode", + "EpisodeIndexRepository", + "Foresight", + "In", + "IndexBackend", + "IndexField", + "IndexFieldKind", + "IndexRepository", + "IndexSchema", + "IsNull", + "KnowledgeTopic", + "ParentType", + "Predicate", + "Scalar", + "UserProfile", + "active_backend", + "agent_case_repo", + "agent_skill_repo", + "all_of", + "any_of", + "atomic_fact_repo", + "compare", + "connect", + "contains", + "drop_business_tables", + "ensure_business_indexes", + "episode_repo", + "eq", + "foresight_repo", + "gt", + "gte", + "is_null", + "knowledge_topic_repo", + "lt", + "lte", + "ne", + "one_of", + "repo_for_schema", + "schema_for", + "shutdown", + "startup", + "user_profile_repo", + "verify_business_schemas", +] diff --git a/src/everos/infra/persistence/index/lancedb.py b/src/everos/infra/persistence/index/lancedb.py new file mode 100644 index 000000000..2b284f571 --- /dev/null +++ b/src/everos/infra/persistence/index/lancedb.py @@ -0,0 +1,470 @@ +"""LanceDB adapter for the backend-neutral derived-index ports.""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import re +from collections.abc import Sequence +from typing import Any, ClassVar, Final + +from lancedb.query import BooleanQuery, FullTextQuery, MatchQuery +from pydantic import BaseModel + +try: + from lancedb.query import Occur +except ImportError: # pragma: no cover + from lancedb._lancedb import Occur # type: ignore[attr-defined,no-redef] + +from everos.component.utils.datetime import ensure_utc, to_iso_format +from everos.core.persistence import LanceRepoBase +from everos.infra.persistence import lancedb as _lancedb + +from .predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, + all_of, + eq, + one_of, +) + +_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_OPERATORS: Final[dict[str, str]] = { + "eq": "=", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate(predicate: Predicate | None) -> str: + """Render a neutral predicate as a LanceDB DataFusion expression.""" + if predicate is None: + return "" + if isinstance(predicate, Comparison): + return ( + f"{_field(predicate.field)} {_OPERATORS[predicate.operator]} " + f"{_literal(predicate.value)}" + ) + if isinstance(predicate, In): + values = ", ".join(_literal(value) for value in predicate.values) + return f"{_field(predicate.field)} IN ({values})" + if isinstance(predicate, Contains): + return f"array_has({_field(predicate.field)}, {_literal(predicate.value)})" + if isinstance(predicate, IsNull): + return f"{_field(predicate.field)} IS NULL" + if isinstance(predicate, All): + return _render_group(predicate.children, "AND") + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "OR") + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _render_group(children: tuple[Predicate, ...], operator: str) -> str: + rendered = [render_predicate(child) for child in children] + rendered = [item for item in rendered if item] + if not rendered: + return "" + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +def _field(value: str) -> str: + if not _FIELD_NAME.fullmatch(value): + raise ValueError(f"invalid predicate field: {value!r}") + return value + + +def _literal(value: Scalar) -> str: + if isinstance(value, str): + return f"'{value.replace(chr(39), chr(39) * 2)}'" + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return f"TIMESTAMP '{to_iso_format(aware)}'" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + return str(value) + + +class LanceIndexRepository[T: BaseModel]: + """Translate the neutral repository contract onto an existing Lance repo.""" + + def __init__(self, repo: LanceRepoBase[T], schema: type[T]) -> None: + self._repo = repo + self.schema = schema + + @property + def table_name(self) -> str: + return str(self.schema.TABLE_NAME) # type: ignore[attr-defined] + + async def add(self, records: Sequence[T]) -> None: + await self._repo.add(records) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + await self._repo.upsert(records, by=by) + + async def count(self) -> int: + return await self._repo.count() + + async def count_where(self, where: Predicate | None = None) -> int: + table = await _lancedb.get_table(self.table_name, self.schema) + return await table.count_rows(filter=_render_optional(where)) + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + return await self._repo.get_by_id(id_value, id_field=id_field) + + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: + return await self._repo.find_where(render_predicate(where), limit=limit) + + async def find_one_where(self, where: Predicate) -> T | None: + return await self._repo.find_one_where(render_predicate(where)) + + async def find_where_paginated( + self, + where: Predicate, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + return await self._repo.find_where_paginated( + render_predicate(where), + sort_by=sort_by, + descending=descending, + page=page, + page_size=page_size, + max_fetch=max_fetch, + ) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Predicate | None = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + if vector is not None: + # Route through dense_search so the metric is pinned to cosine. + # LanceRepoBase.search leaves distance_type at the table default + # (L2), which would not match the Milvus adapter's COSINE. + return await self.dense_search(vector, where, limit=limit) + return await self._repo.search(where=_render_optional(where), limit=limit) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Predicate | None, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + fields = list(columns or getattr(self.schema, "BM25_FIELDS", ())) + clean_terms = [term for term in query_terms if term] + if not clean_terms or not fields: + return [] + table = await _lancedb.get_table(self.table_name, self.schema) + expression = _render_optional(where) + + async def _query_one(field: str) -> list[dict[str, Any]]: + query = build_or_query(clean_terms, field) + assert query is not None + builder = table.query().nearest_to_text(query) + if expression: + builder = builder.where(expression) + return await builder.limit(limit).to_list() + + per_column = await asyncio.gather(*(_query_one(field) for field in fields)) + best: dict[str, dict[str, Any]] = {} + for rows in per_column: + for row in rows: + rid = row.get("id") + if not isinstance(rid, str): + continue + score = float(row.get("_score", 0.0)) + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + shaped = dict(row) + shaped["_score"] = score + best[rid] = shaped + return sorted( + best.values(), + key=lambda row: float(row.get("_score", 0.0)), + reverse=True, + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Predicate | None, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: + if not vector: + return [] + table = await _lancedb.get_table(self.table_name, self.schema) + builder = ( + table.query() + .nearest_to(list(vector)) + .column(vector_field) + .distance_type("cosine") + ) + expression = _render_optional(where) + if expression: + builder = builder.where(expression) + return await builder.limit(limit).to_list() + + async def scan(self, where: Predicate | None = None) -> list[T]: + """Return all matching records without a hidden row cap.""" + table = await _lancedb.get_table(self.table_name, self.schema) + builder = table.query() + expression = _render_optional(where) + if expression: + builder = builder.where(expression) + rows = await builder.to_list() + return [self.schema.model_validate(row) for row in rows] + + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: + await self._repo.update(updates, where=render_predicate(where)) + + async def delete(self, predicate: Predicate) -> None: + await self._repo.delete(render_predicate(predicate)) + + async def delete_by_md_path(self, md_path: str) -> int: + return await self._repo.delete_by_md_path(md_path) + + async def optimize(self) -> None: + await self._repo.optimize() + + async def prune(self, older_than: dt.timedelta) -> None: + await self._repo.prune(older_than) + + async def rebuild_indexes(self) -> None: + await self._repo.rebuild_indexes() + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self.find_where(eq("owner_id", owner_id), limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self.find_one_where(eq("md_path", md_path)) + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self.find_one_where( + all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), + ) + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + if not entry_ids: + return [] + return await self.find_where( + all_of( + eq("owner_id", owner_id), + one_of("entry_id", list(entry_ids)), + eq("app_id", app_id), + eq("project_id", project_id), + ), + limit=len(entry_ids), + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("owner_id", owner_id), eq("session_id", session_id)), + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("parent_type", parent_type), eq("parent_id", parent_id)), + limit=limit, + ) + + +class LanceEpisodeRepository(LanceIndexRepository[_lancedb.Episode]): + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: + return await _lancedb.episode_repo.count_by_owner( + owner_id, + app_id=app_id, + project_id=project_id, + parent_type=parent_type, + ) + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[_lancedb.Episode] | list[dict[str, Any]]: + return await _lancedb.episode_repo.list_by_owner_after_ts( + owner_id=owner_id, + after_ts=after_ts, + parent_type=parent_type, + app_id=app_id, + project_id=project_id, + columns=columns, + limit=limit, + ) + + +class LanceAgentSkillRepository(LanceIndexRepository[_lancedb.AgentSkill]): + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + return await _lancedb.agent_skill_repo.count_in_cluster( + owner_id=owner_id, cluster_id=cluster_id + ) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[_lancedb.AgentSkill]: + return await _lancedb.agent_skill_repo.find_in_cluster( + owner_id=owner_id, cluster_id=cluster_id, limit=limit + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[_lancedb.AgentSkill]: + return await _lancedb.agent_skill_repo.find_topk_relevant_in_cluster( + owner_id=owner_id, + cluster_id=cluster_id, + query_vector=query_vector, + top_k=top_k, + ) + + +episode_repo = LanceEpisodeRepository(_lancedb.episode_repo, _lancedb.Episode) +atomic_fact_repo = LanceIndexRepository(_lancedb.atomic_fact_repo, _lancedb.AtomicFact) +foresight_repo = LanceIndexRepository(_lancedb.foresight_repo, _lancedb.Foresight) +agent_case_repo = LanceIndexRepository(_lancedb.agent_case_repo, _lancedb.AgentCase) +agent_skill_repo = LanceAgentSkillRepository( + _lancedb.agent_skill_repo, _lancedb.AgentSkill +) +user_profile_repo = LanceIndexRepository( + _lancedb.user_profile_repo, _lancedb.UserProfile +) +knowledge_topic_repo = LanceIndexRepository( + _lancedb.knowledge_topic_repo, _lancedb.KnowledgeTopic +) + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + + +class LanceIndexBackend: + """Own LanceDB connection, schema, index, and repository lifecycle.""" + + name: ClassVar[str] = "lancedb" + repositories = ALL_REPOS + + async def connect(self) -> Any: + return await _lancedb.get_connection() + + async def startup(self) -> Any: + connection = await self.connect() + await self.verify_business_schemas() + await self.ensure_business_indexes() + return connection + + async def shutdown(self) -> None: + await _lancedb.dispose_connection() + + async def ensure_business_indexes(self) -> None: + await _lancedb.ensure_business_indexes() + + async def verify_business_schemas(self) -> None: + await _lancedb.verify_business_schemas() + + async def drop_business_tables(self) -> list[str]: + return await _lancedb.drop_business_tables() + + +def _render_optional(predicate: Predicate | None) -> str | None: + rendered = render_predicate(predicate) + return rendered or None + + +def build_or_query(tokens: Sequence[str], column: str) -> FullTextQuery | None: + """Build LanceDB's OR-mode BM25 query for already-tokenized terms.""" + clean = [token for token in tokens if token] + if not clean: + return None + if len(clean) == 1: + return MatchQuery(clean[0], column=column) + return BooleanQuery( + [(Occur.SHOULD, MatchQuery(token, column=column)) for token in clean] + ) + + +lance_index_backend = LanceIndexBackend() + +__all__ = [ + "ALL_REPOS", + "LanceIndexBackend", + "LanceIndexRepository", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "build_or_query", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "lance_index_backend", + "render_predicate", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/index/milvus.py b/src/everos/infra/persistence/index/milvus.py new file mode 100644 index 000000000..f3e1e81a4 --- /dev/null +++ b/src/everos/infra/persistence/index/milvus.py @@ -0,0 +1,56 @@ +"""Milvus lifecycle adapter for the derived-index backend port.""" + +from __future__ import annotations + +from types import ModuleType +from typing import Any, ClassVar + +from .protocols import IndexRepository + + +class MilvusIndexBackend: + """Own remote Milvus connection, collection, and repository lifecycle.""" + + name: ClassVar[str] = "milvus" + + @property + def repositories(self) -> tuple[IndexRepository[Any], ...]: + return tuple(_milvus().ALL_REPOS) + + async def connect(self) -> Any: + return await _milvus().get_client() + + async def startup(self) -> Any: + client = await self.connect() + # ensure_collection creates a missing collection and drift-checks an + # existing one in a single round trip, so unlike the LanceDB backend + # there is no separate verify step to run first. + await self.ensure_business_indexes() + return client + + async def shutdown(self) -> None: + await _milvus().dispose_connection() + + async def ensure_business_indexes(self) -> None: + await _milvus().ensure_business_indexes() + + async def verify_business_schemas(self) -> None: + # Same call as ensure_business_indexes: ensure_collection creates a + # missing collection and drift-checks an existing one. Kept distinct + # because the port and the CLI both address verification by name. + await self.ensure_business_indexes() + + async def drop_business_tables(self) -> list[str]: + return await _milvus().drop_business_tables() + + +def _milvus() -> ModuleType: + # Keep pymilvus genuinely optional for default LanceDB installations. + from everos.infra.persistence import milvus + + return milvus + + +milvus_index_backend = MilvusIndexBackend() + +__all__ = ["MilvusIndexBackend", "milvus_index_backend"] diff --git a/src/everos/infra/persistence/index/predicate.py b/src/everos/infra/persistence/index/predicate.py new file mode 100644 index 000000000..cf5f234fd --- /dev/null +++ b/src/everos/infra/persistence/index/predicate.py @@ -0,0 +1,45 @@ +"""Public predicate API for the derived-index boundary.""" + +from everos.infra.persistence.predicate import All as All +from everos.infra.persistence.predicate import AnyOf as AnyOf +from everos.infra.persistence.predicate import Comparison as Comparison +from everos.infra.persistence.predicate import Contains as Contains +from everos.infra.persistence.predicate import In as In +from everos.infra.persistence.predicate import IsNull as IsNull +from everos.infra.persistence.predicate import Predicate as Predicate +from everos.infra.persistence.predicate import Scalar as Scalar +from everos.infra.persistence.predicate import all_of as all_of +from everos.infra.persistence.predicate import any_of as any_of +from everos.infra.persistence.predicate import compare as compare +from everos.infra.persistence.predicate import contains as contains +from everos.infra.persistence.predicate import eq as eq +from everos.infra.persistence.predicate import gt as gt +from everos.infra.persistence.predicate import gte as gte +from everos.infra.persistence.predicate import is_null as is_null +from everos.infra.persistence.predicate import lt as lt +from everos.infra.persistence.predicate import lte as lte +from everos.infra.persistence.predicate import ne as ne +from everos.infra.persistence.predicate import one_of as one_of + +__all__ = [ + "All", + "AnyOf", + "Comparison", + "Contains", + "In", + "IsNull", + "Predicate", + "Scalar", + "all_of", + "any_of", + "compare", + "contains", + "eq", + "gt", + "gte", + "is_null", + "lt", + "lte", + "ne", + "one_of", +] diff --git a/src/everos/infra/persistence/index/protocols.py b/src/everos/infra/persistence/index/protocols.py new file mode 100644 index 000000000..8a9967bcf --- /dev/null +++ b/src/everos/infra/persistence/index/protocols.py @@ -0,0 +1,199 @@ +"""Typed ports for the rebuildable derived-index subsystem.""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Sequence +from typing import Any, Protocol, TypeVar, runtime_checkable + +from pydantic import BaseModel + +from .predicate import Predicate + +T_co = TypeVar("T_co", bound=BaseModel, covariant=True) +T = TypeVar("T", bound=BaseModel) + + +@runtime_checkable +class IndexRepository(Protocol[T]): + """Backend-neutral repository contract used by memory and cascade. + + Maintenance methods are deliberately part of the contract. Embedded + engines can perform physical work; service-managed engines implement + successful no-ops. This keeps scheduling and health semantics identical + without making the cascade worker know which backend is active. + """ + + schema: type[T] + + @property + def table_name(self) -> str: ... + + async def add(self, records: Sequence[T]) -> None: ... + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: ... + + async def count(self) -> int: ... + + async def count_where(self, where: Predicate | None = None) -> int: ... + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: ... + + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: ... + + async def find_one_where(self, where: Predicate) -> T | None: ... + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: ... + + async def find_by_md_path(self, md_path: str) -> T | None: ... + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: ... + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: ... + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: ... + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: ... + + async def find_where_paginated( + self, + where: Predicate, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: ... + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Predicate | None = None, + limit: int = 10, + ) -> list[dict[str, Any]]: ... + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Predicate | None, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: ... + + async def dense_search( + self, + vector: Sequence[float], + where: Predicate | None, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: ... + + async def scan(self, where: Predicate | None = None) -> list[T]: ... + + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: ... + + async def delete(self, predicate: Predicate) -> None: ... + + async def delete_by_md_path(self, md_path: str) -> int: ... + + async def optimize(self) -> None: ... + + async def prune(self, older_than: dt.timedelta) -> None: ... + + async def rebuild_indexes(self) -> None: ... + + +@runtime_checkable +class EpisodeIndexRepository(IndexRepository[T], Protocol[T]): + """Episode-only reads every backend must also provide.""" + + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: ... + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[Any]: ... + + +@runtime_checkable +class AgentSkillIndexRepository(IndexRepository[T], Protocol[T]): + """AgentSkill-only cluster reads every backend must also provide.""" + + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: ... + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[Any]: ... + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[Any]: ... + + +@runtime_checkable +class IndexBackend(Protocol): + """Lifecycle contract for one configured derived-index backend.""" + + name: str + repositories: tuple[IndexRepository[Any], ...] + + async def connect(self) -> object: ... + + async def startup(self) -> object: ... + + async def shutdown(self) -> None: ... + + async def ensure_business_indexes(self) -> None: ... + + async def verify_business_schemas(self) -> None: ... + + async def drop_business_tables(self) -> list[str]: ... + + +__all__ = [ + "AgentSkillIndexRepository", + "EpisodeIndexRepository", + "IndexBackend", + "IndexRepository", +] diff --git a/src/everos/infra/persistence/index/router.py b/src/everos/infra/persistence/index/router.py new file mode 100644 index 000000000..83480c04d --- /dev/null +++ b/src/everos/infra/persistence/index/router.py @@ -0,0 +1,279 @@ +"""Route stable repository objects to the configured derived-index backend.""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Sequence +from typing import Any, cast + +from pydantic import BaseModel + +from everos.config import load_settings + +from .predicate import Predicate +from .protocols import ( + AgentSkillIndexRepository, + EpisodeIndexRepository, + IndexRepository, +) + + +class RoutedIndexRepository[T: BaseModel]: + """Stable repository identity with a backend selected at call time.""" + + def __init__( + self, + lance_repo: IndexRepository[T], + milvus_repo_name: str, + ) -> None: + self._lance_repo = lance_repo + self._milvus_repo_name = milvus_repo_name + self.schema = lance_repo.schema + + @property + def table_name(self) -> str: + return self._lance_repo.table_name + + def _repo(self) -> IndexRepository[T]: + if load_settings().index.backend == "milvus": + from everos.infra.persistence import milvus + + return getattr(milvus, self._milvus_repo_name) + return self._lance_repo + + async def add(self, records: Sequence[T]) -> None: + await self._repo().add(records) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + await self._repo().upsert(records, by=by) + + async def count(self) -> int: + return await self._repo().count() + + async def count_where(self, where: Predicate | None = None) -> int: + return await self._repo().count_where(where) + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + return await self._repo().get_by_id(id_value, id_field=id_field) + + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: + return await self._repo().find_where(where, limit=limit) + + async def find_one_where(self, where: Predicate) -> T | None: + return await self._repo().find_one_where(where) + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self._repo().find_by_owner(owner_id, limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self._repo().find_by_md_path(md_path) + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self._repo().find_by_owner_entry( + owner_id, + entry_id, + app_id=app_id, + project_id=project_id, + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + return await self._repo().find_by_owner_entries( + owner_id, + entry_ids, + app_id=app_id, + project_id=project_id, + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self._repo().find_by_session( + owner_id, + session_id, + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self._repo().find_by_parent( + parent_type, + parent_id, + limit=limit, + ) + + async def find_where_paginated( + self, + where: Predicate, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + return await self._repo().find_where_paginated( + where, + sort_by=sort_by, + descending=descending, + page=page, + page_size=page_size, + max_fetch=max_fetch, + ) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Predicate | None = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + return await self._repo().search(vector=vector, where=where, limit=limit) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Predicate | None, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + return await self._repo().sparse_search( + query_terms, + where, + columns=columns, + limit=limit, + ) + + async def dense_search( + self, + vector: Sequence[float], + where: Predicate | None, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: + return await self._repo().dense_search( + vector, + where, + limit=limit, + vector_field=vector_field, + ) + + async def scan(self, where: Predicate | None = None) -> list[T]: + return await self._repo().scan(where) + + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: + await self._repo().update(updates, where=where) + + async def delete(self, predicate: Predicate) -> None: + await self._repo().delete(predicate) + + async def delete_by_md_path(self, md_path: str) -> int: + return await self._repo().delete_by_md_path(md_path) + + async def optimize(self) -> None: + await self._repo().optimize() + + async def prune(self, older_than: dt.timedelta) -> None: + await self._repo().prune(older_than) + + async def rebuild_indexes(self) -> None: + await self._repo().rebuild_indexes() + + +class RoutedEpisodeRepository(RoutedIndexRepository[Any]): + def _repo(self) -> EpisodeIndexRepository[Any]: + return cast(EpisodeIndexRepository[Any], super()._repo()) + + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: + return await self._repo().count_by_owner( + owner_id, + app_id=app_id, + project_id=project_id, + parent_type=parent_type, + ) + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[Any]: + return await self._repo().list_by_owner_after_ts( + owner_id=owner_id, + after_ts=after_ts, + parent_type=parent_type, + app_id=app_id, + project_id=project_id, + columns=columns, + limit=limit, + ) + + +class RoutedAgentSkillRepository(RoutedIndexRepository[Any]): + def _repo(self) -> AgentSkillIndexRepository[Any]: + return cast(AgentSkillIndexRepository[Any], super()._repo()) + + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + return await self._repo().count_in_cluster( + owner_id=owner_id, + cluster_id=cluster_id, + ) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[Any]: + return await self._repo().find_in_cluster( + owner_id=owner_id, + cluster_id=cluster_id, + limit=limit, + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[Any]: + return await self._repo().find_topk_relevant_in_cluster( + owner_id=owner_id, + cluster_id=cluster_id, + query_vector=query_vector, + top_k=top_k, + ) + + +__all__ = [ + "RoutedAgentSkillRepository", + "RoutedEpisodeRepository", + "RoutedIndexRepository", +] diff --git a/src/everos/infra/persistence/index/schema.py b/src/everos/infra/persistence/index/schema.py new file mode 100644 index 000000000..0c6178b2b --- /dev/null +++ b/src/everos/infra/persistence/index/schema.py @@ -0,0 +1,149 @@ +"""Logical schema descriptions shared by derived-index adapters. + +The record models remain the source of truth for logical fields. This module +normalizes their supported types into a small backend-neutral vocabulary and +fails loudly when a new field has no portable representation. Physical limits +and index options remain adapter responsibilities. +""" + +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from enum import StrEnum +from functools import cache +from typing import Any, get_args, get_origin + +_DEFAULT_STRING_LENGTH = 65_535 +_ID_LENGTH = 512 +_ARRAY_CAPACITY = 256 + + +class IndexFieldKind(StrEnum): + STRING = "string" + STRING_ARRAY = "string_array" + FLOAT = "float" + INTEGER = "integer" + DATETIME = "datetime" + DENSE_VECTOR = "dense_vector" + + +@dataclass(frozen=True) +class IndexField: + name: str + kind: IndexFieldKind + nullable: bool = False + primary: bool = False + max_length: int | None = None + max_capacity: int | None = None + dimension: int | None = None + + +@dataclass(frozen=True) +class IndexSchema: + table_name: str + model: type[Any] + fields: tuple[IndexField, ...] + bm25_fields: tuple[str, ...] + + def field(self, name: str) -> IndexField: + for field in self.fields: + if field.name == name: + return field + raise KeyError(name) + + @property + def vector_fields(self) -> tuple[IndexField, ...]: + return tuple( + field for field in self.fields if field.kind is IndexFieldKind.DENSE_VECTOR + ) + + @property + def datetime_fields(self) -> frozenset[str]: + return frozenset( + field.name for field in self.fields if field.kind is IndexFieldKind.DATETIME + ) + + +@cache +def schema_for(model: type[Any]) -> IndexSchema: + """Build and validate the portable logical schema for ``model``.""" + table_name = _class_var(model, "TABLE_NAME") + bm25_fields = tuple(_class_var(model, "BM25_FIELDS")) + fields = tuple( + _normalize_field(name, model_field.annotation) + for name, model_field in model.model_fields.items() + ) + declared = {field.name for field in fields} + unknown_bm25 = set(bm25_fields) - declared + if unknown_bm25: + raise ValueError( + f"derived-index schema {table_name!r} has unknown BM25 fields: " + f"{sorted(unknown_bm25)}" + ) + return IndexSchema(table_name, model, fields, bm25_fields) + + +def _normalize_field(name: str, annotation: Any) -> IndexField: + args = get_args(annotation) + optional = type(None) in args + candidates = tuple(arg for arg in args if arg is not type(None)) if optional else () + value_type = candidates[0] if len(candidates) == 1 else annotation + + if value_type is str: + kind = IndexFieldKind.STRING + max_length = _ID_LENGTH if name == "id" else _DEFAULT_STRING_LENGTH + max_capacity = None + dimension = None + elif value_type is float: + kind = IndexFieldKind.FLOAT + max_length = None + max_capacity = None + dimension = None + elif value_type is int: + kind = IndexFieldKind.INTEGER + max_length = None + max_capacity = None + dimension = None + elif value_type is dt.datetime: + kind = IndexFieldKind.DATETIME + max_length = None + max_capacity = None + dimension = None + elif get_origin(value_type) is list and get_args(value_type) == (str,): + kind = IndexFieldKind.STRING_ARRAY + max_length = _ID_LENGTH + max_capacity = _ARRAY_CAPACITY + dimension = None + else: + dimension = getattr(value_type, "dim", None) + if callable(dimension): + dimension = dimension() + if not isinstance(dimension, int) or dimension <= 0: + raise ValueError( + f"derived-index field {name!r} has no portable type mapping: " + f"{annotation!r}" + ) + kind = IndexFieldKind.DENSE_VECTOR + max_length = None + max_capacity = None + + return IndexField( + name=name, + kind=kind, + nullable=optional, + primary=name == "id", + max_length=max_length, + max_capacity=max_capacity, + dimension=dimension, + ) + + +def _class_var(model: type[Any], name: str) -> Any: + value = getattr(model, name, None) + if value is None: + raise ValueError(f"derived-index model {model.__name__} has no {name}") + return value + + +__all__ = ["IndexField", "IndexFieldKind", "IndexSchema", "schema_for"] diff --git a/src/everos/infra/persistence/milvus/__init__.py b/src/everos/infra/persistence/milvus/__init__.py new file mode 100644 index 000000000..d4f80dbdb --- /dev/null +++ b/src/everos/infra/persistence/milvus/__init__.py @@ -0,0 +1,78 @@ +"""Milvus derived index backend. + +This package mirrors the LanceDB business index surface but stores rows in +Milvus collections. It is selected through ``Settings.index.backend`` and is +normally reached through :mod:`everos.infra.persistence.index`. +""" + +from __future__ import annotations + +import asyncio + +from everos.core.observability.logging import get_logger + +from .milvus_manager import MilvusConfigurationError as MilvusConfigurationError +from .milvus_manager import MilvusSchemaMismatchError as MilvusSchemaMismatchError +from .milvus_manager import dispose_connection as dispose_connection +from .milvus_manager import get_client as get_client +from .repos import ALL_REPOS as ALL_REPOS +from .repos import agent_case_repo as agent_case_repo +from .repos import agent_skill_repo as agent_skill_repo +from .repos import atomic_fact_repo as atomic_fact_repo +from .repos import episode_repo as episode_repo +from .repos import foresight_repo as foresight_repo +from .repos import knowledge_topic_repo as knowledge_topic_repo +from .repos import user_profile_repo as user_profile_repo +from .repository import MilvusRepoBase as MilvusRepoBase +from .repository import MilvusValueLimitError as MilvusValueLimitError + +logger = get_logger(__name__) + + +async def ensure_business_indexes() -> None: + """Create or verify every EverOS Milvus collection.""" + for repo in ALL_REPOS: + await repo.ensure_collection() + + +async def drop_business_tables() -> list[str]: + """Drop every configured Milvus collection and return their names.""" + client = await get_client() + dropped: list[str] = [] + for repo in ALL_REPOS: + name = repo.collection_name + if await asyncio.to_thread(client.has_collection, name): + try: + await asyncio.to_thread(client.drop_collection, name) + except Exception: + # Zilliz Serverless can complete the drop server-side while + # its gateway returns DEADLINE_EXCEEDED. Confirm state before + # turning an already-successful rebuild/cleanup into failure. + if await asyncio.to_thread(client.has_collection, name): + raise + logger.warning( + "milvus_collection_drop_confirmed_after_client_error", + collection=name, + ) + dropped.append(name) + MilvusRepoBase._reset_collection_cache() + return dropped + + +__all__ = [ + "ALL_REPOS", + "MilvusConfigurationError", + "MilvusSchemaMismatchError", + "MilvusValueLimitError", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "dispose_connection", + "drop_business_tables", + "ensure_business_indexes", + "episode_repo", + "foresight_repo", + "get_client", + "knowledge_topic_repo", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/milvus/milvus_manager.py b/src/everos/infra/persistence/milvus/milvus_manager.py new file mode 100644 index 000000000..05af230b4 --- /dev/null +++ b/src/everos/infra/persistence/milvus/milvus_manager.py @@ -0,0 +1,105 @@ +"""Milvus connection and collection management for the derived index.""" + +from __future__ import annotations + +import re + +from pymilvus import MilvusClient + +from everos.config import MilvusSettings, load_settings +from everos.core.errors import ConfigurationError +from everos.core.observability.logging import get_logger + +logger = get_logger(__name__) + +_client: MilvusClient | None = None + + +class MilvusSchemaMismatchError(RuntimeError): + """Raised when an existing Milvus collection does not match EverOS.""" + + +class MilvusConfigurationError(ConfigurationError): + """Raised when the remote Milvus profile is incomplete or invalid.""" + + +def collection_name(table_name: str, settings: MilvusSettings | None = None) -> str: + """Return the configured Milvus collection name for an EverOS table.""" + cfg = settings or load_settings().milvus + prefix = _sanitize_name_part(cfg.collection_prefix) + base = _sanitize_name_part(table_name) + name = f"{prefix}_{base}" if prefix else base + if not re.match(r"^[A-Za-z_]", name): + name = f"_{name}" + return name + + +async def get_client() -> MilvusClient: + """Return the process-wide MilvusClient, creating it lazily.""" + global _client + if _client is None: + settings = load_settings().milvus + uri = _resolve_uri(settings) + token = _secret(settings.token) + db_name = settings.db_name or "" + _client = MilvusClient(uri=uri, token=token, db_name=db_name) + logger.info( + "milvus_connection_opened", + uri=uri, + db_name=db_name or None, + consistency_level=settings.consistency_level, + ) + return _client + + +async def dispose_connection() -> None: + """Close the process-wide Milvus client.""" + global _client + if _client is not None: + _client.close() + _client = None + logger.info("milvus_connection_closed") + from .repository import MilvusRepoBase + + MilvusRepoBase._reset_collection_cache() + + +def _resolve_uri(settings: MilvusSettings) -> str: + uri = settings.uri.strip() + if not uri: + raise MilvusConfigurationError( + "[index] backend = 'milvus' requires EVEROS_MILVUS__URI (or " + "[milvus] uri) pointing to Milvus Server or Zilliz Cloud; " + "embedded Milvus Lite is not supported" + ) + scheme, separator, _rest = uri.partition("://") + if not separator or scheme.lower() not in {"http", "https"}: + raise MilvusConfigurationError( + "[milvus] uri must be a remote http(s) endpoint for Milvus Server " + "or Zilliz Cloud, not a local database path; embedded Milvus Lite " + "is not supported" + ) + return uri + + +def _secret(value: object | None) -> str: + if value is None: + return "" + getter = getattr(value, "get_secret_value", None) + if callable(getter): + return getter() or "" + return str(value) + + +def _sanitize_name_part(value: str) -> str: + clean = re.sub(r"\W+", "_", value.strip()) + return clean.strip("_") + + +__all__ = [ + "MilvusConfigurationError", + "MilvusSchemaMismatchError", + "collection_name", + "dispose_connection", + "get_client", +] diff --git a/src/everos/infra/persistence/milvus/predicate.py b/src/everos/infra/persistence/milvus/predicate.py new file mode 100644 index 000000000..785e77605 --- /dev/null +++ b/src/everos/infra/persistence/milvus/predicate.py @@ -0,0 +1,109 @@ +"""Render backend-neutral predicates as Milvus filter expressions.""" + +from __future__ import annotations + +import datetime as dt +import json +import re +from collections.abc import Collection +from typing import Final + +from everos.component.utils.datetime import ensure_utc, to_timestamp_ms +from everos.infra.persistence.predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, +) + +_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_OPERATORS: Final[dict[str, str]] = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate( + predicate: Predicate | None, + *, + datetime_fields: Collection[str] = (), + vector_fields: Collection[str] = (), +) -> str: + """Render a predicate using Milvus operators and physical field names.""" + if predicate is None: + return "" + if isinstance(predicate, Comparison): + return ( + f"{_field(predicate.field, datetime_fields)} " + f"{_OPERATORS[predicate.operator]} {_literal(predicate.value)}" + ) + if isinstance(predicate, In): + values = ", ".join(_literal(value) for value in predicate.values) + return f"{_field(predicate.field, datetime_fields)} in [{values}]" + if isinstance(predicate, Contains): + return ( + f"array_contains({_field(predicate.field, datetime_fields)}, " + f"{_literal(predicate.value)})" + ) + if isinstance(predicate, IsNull): + if predicate.field in vector_fields: + return f"{_field(predicate.field, ())}__present == false" + return f"{_field(predicate.field, datetime_fields)} is null" + if isinstance(predicate, All): + return _render_group(predicate.children, "and", datetime_fields, vector_fields) + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "or", datetime_fields, vector_fields) + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _render_group( + children: tuple[Predicate, ...], + operator: str, + datetime_fields: Collection[str], + vector_fields: Collection[str], +) -> str: + rendered = [ + render_predicate( + child, + datetime_fields=datetime_fields, + vector_fields=vector_fields, + ) + for child in children + ] + rendered = [item for item in rendered if item] + if not rendered: + return "" + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +def _field(value: str, datetime_fields: Collection[str]) -> str: + if not _FIELD_NAME.fullmatch(value): + raise ValueError(f"invalid predicate field: {value!r}") + if value in datetime_fields: + return f"{value}_ms" + return value + + +def _literal(value: Scalar) -> str: + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return str(to_timestamp_ms(aware)) + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +__all__ = ["render_predicate"] diff --git a/src/everos/infra/persistence/milvus/repos.py b/src/everos/infra/persistence/milvus/repos.py new file mode 100644 index 000000000..a43095af6 --- /dev/null +++ b/src/everos/infra/persistence/milvus/repos.py @@ -0,0 +1,165 @@ +"""Milvus repo singletons for EverOS derived index tables.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from everos.component.utils.datetime import from_timestamp +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + UserProfile, +) +from everos.infra.persistence.predicate import all_of, eq, gt, is_null + +from .repository import MilvusRepoBase + + +class _EpisodeRepo(MilvusRepoBase[Episode]): + schema = Episode + + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: + return await self._count_where( + all_of( + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + eq("parent_type", parent_type) if parent_type is not None else None, + ) + ) + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[Episode] | list[dict[str, Any]]: + predicate = all_of( + eq("owner_id", owner_id), + gt("timestamp", from_timestamp(after_ts)), + eq("parent_type", parent_type), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + ) + # A plain query(limit=...) cannot cross Milvus' 16,384-row window, so + # the ceiling has to be applied through the iterator instead. + raw = await self._scan_raw( + predicate, include_vectors=True, max_rows=limit or 20_000 + ) + rows = [self._model_from_milvus(row) for row in raw] + rows.sort(key=lambda row: row.timestamp) + if columns is None: + return rows + projection = list(dict.fromkeys([*columns, "timestamp"])) + return [{name: getattr(row, name) for name in projection} for row in rows] + + +class _AtomicFactRepo(MilvusRepoBase[AtomicFact]): + schema = AtomicFact + + +class _ForesightRepo(MilvusRepoBase[Foresight]): + schema = Foresight + + +class _AgentCaseRepo(MilvusRepoBase[AgentCase]): + schema = AgentCase + + +class _AgentSkillRepo(MilvusRepoBase[AgentSkill]): + schema = AgentSkill + + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + return await self._count_where( + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)) + ) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[AgentSkill]: + return await self.find_where( + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), + limit=limit, + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[AgentSkill]: + if not query_vector: + raise ValueError( + "query_vector must be non-empty; " + "call find_in_cluster for the scalar fallback" + ) + rows = await self.dense_search( + query_vector, + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), + limit=top_k, + ) + out: list[AgentSkill] = [] + for row in rows: + rid = row.get("id") + if isinstance(rid, str) and (item := await self.get_by_id(rid)) is not None: + out.append(item) + return out + + +class _UserProfileRepo(MilvusRepoBase[UserProfile]): + schema = UserProfile + + +class _KnowledgeTopicRepo(MilvusRepoBase[KnowledgeTopic]): + schema = KnowledgeTopic + + +episode_repo = _EpisodeRepo() +atomic_fact_repo = _AtomicFactRepo() +foresight_repo = _ForesightRepo() +agent_case_repo = _AgentCaseRepo() +agent_skill_repo = _AgentSkillRepo() +user_profile_repo = _UserProfileRepo() +knowledge_topic_repo = _KnowledgeTopicRepo() + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py new file mode 100644 index 000000000..fb9ce713f --- /dev/null +++ b/src/everos/infra/persistence/milvus/repository.py @@ -0,0 +1,948 @@ +"""Milvus repository for EverOS rebuildable derived indexes.""" + +from __future__ import annotations + +import asyncio +import datetime as dt +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any, ClassVar + +from pydantic import BaseModel +from pymilvus import DataType, Function, FunctionType, MilvusClient + +from everos.component.utils.datetime import ( + ensure_utc, + from_timestamp_ms, + to_timestamp_ms, +) +from everos.config import load_settings +from everos.core.observability.logging import get_logger +from everos.infra.persistence.index.schema import ( + IndexField, + IndexFieldKind, + IndexSchema, + schema_for, +) +from everos.infra.persistence.predicate import ( + Predicate, + all_of, + eq, + one_of, +) + +from .milvus_manager import MilvusSchemaMismatchError, collection_name, get_client +from .predicate import render_predicate + +logger = get_logger(__name__) + +_DUMMY_VECTOR_FIELD = "_everos_dummy_vector" +_DUMMY_VECTOR_DIMENSION = 2 +_SPARSE_SUFFIX = "__sparse" +_PRESENT_SUFFIX = "__present" + + +class MilvusValueLimitError(ValueError): + """A row exceeds a documented Milvus VARCHAR, array, or vector limit.""" + + +@dataclass(frozen=True) +class _PhysicalField: + """One Milvus column exactly as EverOS declares it. + + The same descriptor drives collection creation and startup verification, + so a reported mismatch always means the server disagrees with us — never + that the builder and the checker have drifted apart from each other. + """ + + name: str + datatype: DataType + is_primary: bool = False + nullable: bool = False + dim: int | None = None + element_type: DataType | None = None + max_length: int | None = None + max_capacity: int | None = None + enable_analyzer: bool = False + + def create_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = {"field_name": self.name, "datatype": self.datatype} + if self.is_primary: + kwargs["is_primary"] = True + if self.element_type is not None: + kwargs["element_type"] = self.element_type + if self.max_length is not None: + kwargs["max_length"] = self.max_length + if self.max_capacity is not None: + kwargs["max_capacity"] = self.max_capacity + if self.dim is not None: + kwargs["dim"] = self.dim + if self.enable_analyzer: + kwargs["enable_analyzer"] = True + if self.nullable: + kwargs["nullable"] = True + return kwargs + + def mismatches(self, actual: dict[str, Any]) -> list[str]: + """Report how a server-reported field differs from this declaration. + + ``describe_collection`` omits ``nullable`` and ``is_primary`` when they + are false and omits ``element_type`` for non-array columns, so absence + is unambiguous and can be compared strictly. ``max_length`` and + ``max_capacity`` are reported only for the types that carry them and + the server may normalize them, so they are advisory: a wrong length + surfaces as a loud write rejection anyway, whereas a wrong datatype or + dimension is the kind that fails opaquely much later. + """ + params = actual.get("params") or {} + out: list[str] = [] + if actual.get("type") != self.datatype: + out.append( + f"{self.name}: datatype {_name_of(actual.get('type'))} " + f"!= expected {_name_of(self.datatype)}" + ) + if bool(actual.get("is_primary", False)) != self.is_primary: + out.append( + f"{self.name}: is_primary {actual.get('is_primary', False)} " + f"!= expected {self.is_primary}" + ) + if bool(actual.get("nullable", False)) != self.nullable: + out.append( + f"{self.name}: nullable {actual.get('nullable', False)} " + f"!= expected {self.nullable}" + ) + if self.dim is not None and params.get("dim") != self.dim: + out.append(f"{self.name}: dim {params.get('dim')} != expected {self.dim}") + if self.element_type is not None and actual.get("element_type") != ( + self.element_type + ): + out.append( + f"{self.name}: element_type {_name_of(actual.get('element_type'))} " + f"!= expected {_name_of(self.element_type)}" + ) + return out + + def soft_mismatches(self, actual: dict[str, Any]) -> list[str]: + """Advisory-only limit drift (see :meth:`mismatches`).""" + params = actual.get("params") or {} + out: list[str] = [] + for key, want in ( + ("max_length", self.max_length), + ("max_capacity", self.max_capacity), + ): + got = params.get(key) + if want is not None and got is not None and int(got) != want: + out.append(f"{self.name}: {key} {got} != declared {want}") + return out + + +def _name_of(datatype: Any) -> str: + return getattr(datatype, "name", repr(datatype)) + + +class MilvusRepoBase[T: BaseModel]: + """Generic Milvus repository backed by one neutral index schema.""" + + schema: type[T] + _write_locks: ClassVar[dict[str, asyncio.Lock]] = {} + _collection_locks: ClassVar[dict[str, asyncio.Lock]] = {} + _ready_collections: ClassVar[set[str]] = set() + + @property + def index_schema(self) -> IndexSchema: + return schema_for(self.schema) + + @property + def table_name(self) -> str: + return self.index_schema.table_name + + @property + def collection_name(self) -> str: + return collection_name(self.table_name) + + @classmethod + def _write_lock(cls, name: str) -> asyncio.Lock: + return cls._write_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _collection_lock(cls, name: str) -> asyncio.Lock: + return cls._collection_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _reset_collection_cache(cls) -> None: + # Only the readiness set is cleared. Dropping _collection_locks would + # hand a fresh Lock to the next caller while another task still holds + # the old one, so ensure_collection would stop being mutually + # exclusive exactly when a drop/rebuild is in flight. + cls._ready_collections.clear() + + @classmethod + def _reset_locks_for_tests(cls) -> None: + cls._write_locks.clear() + cls._collection_locks.clear() + cls._reset_collection_cache() + + async def ensure_collection(self) -> None: + """Create or verify the collection once per process.""" + name = self.collection_name + if name in self._ready_collections: + return + async with self._collection_lock(name): + if name in self._ready_collections: + return + client = await get_client() + if await _run(client.has_collection, name): + await self.verify_collection() + else: + await self._create_collection(client) + self._ready_collections.add(name) + + async def _create_collection(self, client: MilvusClient) -> None: + schema = self._build_collection_schema() + index_params = client.prepare_index_params() + vector_fields = self.index_schema.vector_fields + if vector_fields: + for field in vector_fields: + index_params.add_index( + field_name=field.name, + index_type="AUTOINDEX", + metric_type="COSINE", + ) + else: + index_params.add_index( + field_name=_DUMMY_VECTOR_FIELD, + index_type="AUTOINDEX", + metric_type="COSINE", + ) + for field in self.index_schema.bm25_fields: + index_params.add_index( + field_name=_sparse_field(field), + index_type="AUTOINDEX", + metric_type="BM25", + ) + settings = load_settings().milvus + await _run( + client.create_collection, + collection_name=self.collection_name, + schema=schema, + index_params=index_params, + consistency_level=settings.consistency_level, + ) + logger.info( + "milvus_collection_created", + table=self.table_name, + collection=self.collection_name, + ) + + async def verify_collection(self) -> None: + """Reject a collection whose physical schema disagrees with ours. + + A name-only check waves through a collection whose columns happen to + share our names but carry the wrong datatype, primary key, nullability + or vector dimension — a stale model, or a ``collection_prefix`` + collision with someone else's data. That collection starts fine and + then fails opaquely on the first write or search. Comparing the + physical shape turns it back into a startup error with a recovery + path. + """ + client = await get_client() + description = await _run(client.describe_collection, self.collection_name) + reported = { + field["name"]: field + for field in description.get("fields", []) + if "name" in field + } + expected = {physical.name: physical for physical in self._physical_fields()} + + missing = sorted(set(expected) - set(reported)) + stale = sorted(set(reported) - set(expected)) + drift: list[str] = [] + advisory: list[str] = [] + for name, physical in expected.items(): + actual = reported.get(name) + if actual is None: + continue + drift.extend(physical.mismatches(actual)) + advisory.extend(physical.soft_mismatches(actual)) + + if advisory: + logger.warning( + "milvus_collection_limit_drift", + collection=self.collection_name, + details=advisory, + ) + if missing or stale or drift: + raise MilvusSchemaMismatchError( + f"Milvus collection {self.collection_name!r} schema drift: " + f"missing={missing}, stale={stale}, incompatible={drift}. " + "The index is rebuildable from markdown; run " + "`everos cascade rebuild`." + ) + + async def add(self, records: Sequence[T]) -> None: + if not records: + return + await self.ensure_collection() + payload = [self._to_milvus_record(record) for record in records] + client = await get_client() + async with self._write_lock(self.collection_name): + await _run(client.insert, self.collection_name, payload) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + if by != "id": + raise ValueError("MilvusRepoBase only supports upsert by id") + if not records: + return + await self.ensure_collection() + payload = [self._to_milvus_record(record) for record in records] + client = await get_client() + async with self._write_lock(self.collection_name): + await _run(client.upsert, self.collection_name, payload) + + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: + """Read-modify-write the matching rows. + + Milvus has no partial-column update, so the whole row is read back and + re-upserted. Both halves must hold the same lock: with the read + outside it, two concurrent updates to one row (backfill writing + ``vector`` while reflection writes ``deprecated_by``) each overwrite + the other's column with the value they read before it landed. + """ + client = await get_client() + async with self._write_lock(self.collection_name): + rows = await self._scan_raw(where, include_vectors=True) + if not rows: + return + patched: list[dict[str, Any]] = [] + for row in rows: + merged = dict(row) + for key, value in updates.items(): + self._write_field_value(merged, key, value) + self._validate_raw_record(merged) + patched.append(merged) + await _run(client.upsert, self.collection_name, patched) + + async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None: + """Milvus indexes and compaction are service-managed.""" + + async def prune(self, older_than: dt.timedelta) -> None: + """Milvus compaction and retention are service-managed.""" + + async def rebuild_indexes(self) -> None: + """Milvus AUTOINDEX maintenance is service-managed.""" + + async def count(self) -> int: + return await self._count_where(None) + + async def count_where(self, where: Predicate | None = None) -> int: + return await self._count_where(where) + + async def _count_where(self, where: Predicate | None) -> int: + await self.ensure_collection() + client = await get_client() + rows = await _run( + client.query, + self.collection_name, + filter=self._expr(where), + output_fields=["count(*)"], + ) + return int(rows[0].get("count(*)", 0)) if rows else 0 + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + if id_field != "id": + rows = await self.find_where(eq(id_field, id_value), limit=1) + return rows[0] if rows else None + await self.ensure_collection() + client = await get_client() + rows = await _run( + client.get, + self.collection_name, + ids=[id_value], + output_fields=self._output_fields(include_vectors=True), + ) + return self._model_from_milvus(rows[0]) if rows else None + + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: + rows = await self._query_raw(where, limit=limit, include_vectors=True) + return [self._model_from_milvus(row) for row in rows] + + async def find_one_where(self, where: Predicate) -> T | None: + rows = await self.find_where(where, limit=1) + return rows[0] if rows else None + + async def scan(self, where: Predicate | None = None) -> list[T]: + """Stream every matching row without a fixed query-window cap.""" + rows = await self._scan_raw(where, include_vectors=True) + return [self._model_from_milvus(row) for row in rows] + + async def _scan_raw( + self, + where: Predicate | None, + *, + include_vectors: bool, + max_rows: int | None = None, + ) -> list[dict[str, Any]]: + """Read through Milvus' iterator, optionally stopping at a soft cap. + + A normal ``query(limit=...)`` cannot cross Milvus' 16,384-row result + window. The public pagination contract deliberately allows a 20,000 + candidate window, so both maintenance scans and paginated reads must + use the iterator API. + """ + await self.ensure_collection() + client = await get_client() + iterator = await _run( + client.query_iterator, + self.collection_name, + batch_size=1000, + limit=-1, + filter=self._expr(where), + output_fields=self._output_fields(include_vectors=include_vectors), + ) + rows: list[dict[str, Any]] = [] + try: + while batch := await _run(iterator.next): + rows.extend(batch) + if max_rows is not None and len(rows) >= max_rows: + del rows[max_rows:] + break + finally: + await _run(iterator.close) + return rows + + async def find_where_paginated( + self, + where: Predicate, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + total = await self._count_where(where) + raw = await self._scan_raw( + where, + include_vectors=True, + max_rows=max_fetch, + ) + if total > len(raw): + logger.warning( + "milvus_find_where_paginated_truncated", + table=self.table_name, + total=total, + max_fetch=max_fetch, + ) + rows = [self._model_from_milvus(row) for row in raw] + rows.sort( + key=lambda row: _sort_value(getattr(row, sort_by, None)), + reverse=descending, + ) + offset = (page - 1) * page_size + return rows[offset : offset + page_size], total + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self.find_where(eq("owner_id", owner_id), limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self.find_one_where(eq("md_path", md_path)) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Predicate | None = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + if vector is None: + return await self._query_candidate_rows(where, limit=limit) + return await self.dense_search(vector, where, limit=limit) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Predicate | None, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + if not query_terms: + return [] + await self.ensure_collection() + fields = list(columns or self.index_schema.bm25_fields) + if not fields: + return [] + unknown = set(fields) - set(self.index_schema.bm25_fields) + if unknown: + raise ValueError(f"unknown BM25 fields: {sorted(unknown)}") + client = await get_client() + query = " ".join(term for term in query_terms if term) + best: dict[str, dict[str, Any]] = {} + for field in fields: + results = await _run( + client.search, + self.collection_name, + data=[query], + anns_field=_sparse_field(field), + filter=self._expr(where), + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "BM25"}, + ) + for row in _first_result_set(results): + shaped = self._candidate_row_from_search(row) + score = _bm25_score_from_distance(row.get("distance")) + shaped["_score"] = score + rid = shaped.get("id") + if not isinstance(rid, str): + continue + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + best[rid] = shaped + return sorted( + best.values(), key=lambda item: float(item.get("_score", 0.0)), reverse=True + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Predicate | None, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: + if not vector: + return [] + field = self.index_schema.field(vector_field) + if field.kind is not IndexFieldKind.DENSE_VECTOR: + raise ValueError(f"{vector_field!r} is not a dense-vector field") + self._validate_vector(field, vector) + await self.ensure_collection() + client = await get_client() + present = eq(_present_field(vector_field), True) + results = await _run( + client.search, + self.collection_name, + data=[list(vector)], + anns_field=vector_field, + filter=self._expr(all_of(where, present)), + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "COSINE"}, + ) + return [ + self._candidate_row_from_search(row, normalize_cosine=True) + for row in _first_result_set(results) + ] + + async def delete(self, predicate: Predicate) -> None: + await self.ensure_collection() + client = await get_client() + async with self._write_lock(self.collection_name): + await _run( + client.delete, + self.collection_name, + filter=self._expr(predicate), + ) + + async def delete_by_md_path(self, md_path: str) -> int: + predicate = eq("md_path", md_path) + count = await self._count_where(predicate) + if count: + await self.delete(predicate) + return count + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self.find_one_where( + all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), + ) + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + if not entry_ids: + return [] + return await self.find_where( + all_of( + eq("owner_id", owner_id), + one_of("entry_id", list(entry_ids)), + eq("app_id", app_id), + eq("project_id", project_id), + ), + limit=len(entry_ids), + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("owner_id", owner_id), eq("session_id", session_id)), + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("parent_type", parent_type), eq("parent_id", parent_id)), + limit=limit, + ) + + def _build_collection_schema(self) -> Any: + schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False) + for physical in self._physical_fields(): + schema.add_field(**physical.create_kwargs()) + for field in self.index_schema.bm25_fields: + sparse_name = _sparse_field(field) + schema.add_function( + Function( + name=f"{field}_bm25", + function_type=FunctionType.BM25, + input_field_names=[field], + output_field_names=[sparse_name], + ) + ) + return schema + + def _physical_fields(self) -> tuple[_PhysicalField, ...]: + """Every Milvus column this table declares, in creation order.""" + out: list[_PhysicalField] = [] + for field in self.index_schema.fields: + out.extend(self._physical_for(field)) + if not self.index_schema.vector_fields: + out.append( + _PhysicalField( + name=_DUMMY_VECTOR_FIELD, + datatype=DataType.FLOAT_VECTOR, + dim=_DUMMY_VECTOR_DIMENSION, + ) + ) + out.extend( + _PhysicalField( + name=_sparse_field(field), + datatype=DataType.SPARSE_FLOAT_VECTOR, + ) + for field in self.index_schema.bm25_fields + ) + return tuple(out) + + def _physical_for(self, field: IndexField) -> list[_PhysicalField]: + is_bm25 = field.name in self.index_schema.bm25_fields + if field.kind is IndexFieldKind.STRING: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.VARCHAR, + is_primary=field.primary, + # A BM25 input is written as "" rather than null so the + # analyzer always has something to tokenize. + nullable=field.nullable and not field.primary and not is_bm25, + max_length=field.max_length, + enable_analyzer=is_bm25, + ) + ] + if field.kind is IndexFieldKind.STRING_ARRAY: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.ARRAY, + nullable=field.nullable, + element_type=DataType.VARCHAR, + max_length=field.max_length, + max_capacity=field.max_capacity, + ) + ] + if field.kind is IndexFieldKind.FLOAT: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.DOUBLE, + nullable=field.nullable, + ) + ] + if field.kind is IndexFieldKind.INTEGER: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.INT64, + nullable=field.nullable, + ) + ] + if field.kind is IndexFieldKind.DATETIME: + return [ + _PhysicalField( + name=_datetime_storage_field(field.name), + datatype=DataType.INT64, + nullable=field.nullable, + ) + ] + if field.kind is IndexFieldKind.DENSE_VECTOR: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.FLOAT_VECTOR, + dim=field.dimension, + ), + # Milvus cannot store a null dense vector, so a logical null is + # a zero vector plus this presence marker. + _PhysicalField( + name=_present_field(field.name), + datatype=DataType.BOOL, + ), + ] + # pragma: no cover - enum exhaustiveness guard + raise TypeError(f"unsupported index field kind: {field.kind}") + + def _stored_field_names(self) -> list[str]: + return [physical.name for physical in self._physical_fields()] + + def _output_fields(self, *, include_vectors: bool) -> list[str]: + fields: list[str] = [] + vector_names = {field.name for field in self.index_schema.vector_fields} + present_names = {_present_field(name) for name in vector_names} + for name in self._stored_field_names(): + if name.endswith(_SPARSE_SUFFIX): + continue + if name == _DUMMY_VECTOR_FIELD: + if include_vectors: + fields.append(name) + continue + if not include_vectors and (name in vector_names or name in present_names): + continue + fields.append(name) + return fields + + def _to_milvus_record(self, record: T) -> dict[str, Any]: + raw = record.model_dump(mode="python") + out: dict[str, Any] = {} + for field in self.index_schema.fields: + value = raw.get(field.name) + if field.name in self.index_schema.bm25_fields and value is None: + value = "" + if field.kind is IndexFieldKind.DATETIME: + out[_datetime_storage_field(field.name)] = ( + _datetime_to_ms(value) if value is not None else None + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + present = value is not None + out[field.name] = ( + list(value) if present else [0.0] * int(field.dimension or 0) + ) + out[_present_field(field.name)] = present + elif field.kind is IndexFieldKind.STRING_ARRAY: + out[field.name] = [str(item) for item in (value or [])] + else: + out[field.name] = value + if not self.index_schema.vector_fields: + out[_DUMMY_VECTOR_FIELD] = [0.0] * _DUMMY_VECTOR_DIMENSION + self._validate_raw_record(out) + return out + + def _validate_raw_record(self, row: dict[str, Any]) -> None: + for field in self.index_schema.fields: + storage_name = ( + _datetime_storage_field(field.name) + if field.kind is IndexFieldKind.DATETIME + else field.name + ) + value = row.get(storage_name) + if value is None: + if not field.nullable: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} cannot be null" + ) + continue + if field.kind is IndexFieldKind.STRING: + size = len(str(value).encode("utf-8")) + if field.max_length is not None and size > field.max_length: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} is {size} UTF-8 bytes; " + f"Milvus limit is {field.max_length}" + ) + elif field.kind is IndexFieldKind.STRING_ARRAY: + if field.max_capacity is not None and len(value) > field.max_capacity: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} has {len(value)} items; " + f"Milvus limit is {field.max_capacity}" + ) + for position, item in enumerate(value): + size = len(str(item).encode("utf-8")) + if field.max_length is not None and size > field.max_length: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name}[{position}] is {size} " + f"UTF-8 bytes; Milvus limit is {field.max_length}" + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + self._validate_vector(field, value) + + def _validate_vector(self, field: IndexField, value: Sequence[float]) -> None: + if len(value) != field.dimension: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} has dimension {len(value)}; " + f"expected {field.dimension}" + ) + + def _model_from_milvus(self, row: dict[str, Any]) -> T: + return self.schema.model_validate(self._restore_row(row)) + + def _candidate_row_from_search( + self, row: dict[str, Any], *, normalize_cosine: bool = False + ) -> dict[str, Any]: + shaped = self._restore_row(row.get("entity", {})) + raw_distance = row.get("distance") + shaped["_distance"] = ( + _cosine_distance_from_milvus(raw_distance) + if normalize_cosine + else raw_distance + ) + return shaped + + def _restore_row(self, row: dict[str, Any]) -> dict[str, Any]: + out: dict[str, Any] = {} + for field in self.index_schema.fields: + if field.kind is IndexFieldKind.DATETIME: + value = row.get(_datetime_storage_field(field.name)) + out[field.name] = ( + None if value is None else from_timestamp_ms(int(value)) + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + if field.name in row: + out[field.name] = ( + row[field.name] + if row.get(_present_field(field.name), True) + else None + ) + elif field.name in row: + value = row[field.name] + if ( + field.name in self.index_schema.bm25_fields + and value == "" + and field.nullable + ): + value = None + out[field.name] = value + return out + + def _write_field_value( + self, row: dict[str, Any], field_name: str, value: Any + ) -> None: + field = self.index_schema.field(field_name) + if field.kind is IndexFieldKind.DATETIME: + row[_datetime_storage_field(field_name)] = ( + _datetime_to_ms(value) if value is not None else None + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + present = value is not None + row[field_name] = ( + list(value) if present else [0.0] * int(field.dimension or 0) + ) + row[_present_field(field_name)] = present + else: + row[field_name] = value + + async def _query_raw( + self, + where: Predicate | None, + *, + limit: int, + include_vectors: bool, + ) -> list[dict[str, Any]]: + await self.ensure_collection() + client = await get_client() + return await _run( + client.query, + self.collection_name, + filter=self._expr(where), + output_fields=self._output_fields(include_vectors=include_vectors), + limit=limit, + ) + + async def _query_candidate_rows( + self, where: Predicate | None, *, limit: int + ) -> list[dict[str, Any]]: + rows = await self._query_raw(where, limit=limit, include_vectors=False) + return [self._restore_row(row) for row in rows] + + def _expr(self, where: Predicate | None) -> str: + if where is not None and not isinstance(where, Predicate): + raise TypeError( + "Milvus repository predicates must use the neutral Predicate AST, " + f"got {type(where).__name__}" + ) + return render_predicate( + where, + datetime_fields=self.index_schema.datetime_fields, + vector_fields={field.name for field in self.index_schema.vector_fields}, + ) + + +def _datetime_storage_field(name: str) -> str: + return f"{name}_ms" + + +def _present_field(name: str) -> str: + return f"{name}{_PRESENT_SUFFIX}" + + +def _sparse_field(name: str) -> str: + return f"{name}{_SPARSE_SUFFIX}" + + +def _datetime_to_ms(value: Any) -> int: + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return to_timestamp_ms(aware) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(value) + raise TypeError(f"expected datetime or epoch ms, got {type(value).__name__}") + + +def _sort_value(value: Any) -> Any: + if value is None: + fallback = ensure_utc(dt.datetime.min) + assert fallback is not None + return fallback + return value + + +def _bm25_score_from_distance(distance: Any) -> float: + """Milvus BM25 is higher-is-better; expose a non-negative score.""" + return 0.0 if distance is None else max(0.0, float(distance)) + + +def _cosine_distance_from_milvus(distance: Any) -> float | None: + """Convert Milvus Server / Zilliz similarity to Lance-style distance.""" + if distance is None: + return None + return min(1.0, max(0.0, 1.0 - float(distance))) + + +def _first_result_set(results: Any) -> list[dict[str, Any]]: + if not results: + return [] + return list(results[0] or []) + + +async def _run(func: Any, /, *args: Any, **kwargs: Any) -> Any: + return await asyncio.to_thread(func, *args, **kwargs) + + +__all__ = ["MilvusRepoBase", "MilvusValueLimitError"] diff --git a/src/everos/infra/persistence/predicate.py b/src/everos/infra/persistence/predicate.py new file mode 100644 index 000000000..0ca2da0a3 --- /dev/null +++ b/src/everos/infra/persistence/predicate.py @@ -0,0 +1,159 @@ +"""Backend-neutral predicate tree for rebuildable derived indexes. + +Application and memory code construct these nodes. A storage adapter owns +the rendering into its physical query language, so backend syntax never leaks +above :mod:`everos.infra.persistence`. +""" + +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from typing import Literal + +type Scalar = str | int | float | bool | dt.datetime +type ComparisonOperator = Literal["eq", "ne", "gt", "gte", "lt", "lte"] + + +class Predicate: + """Marker base class for derived-index predicates.""" + + +@dataclass(frozen=True) +class Comparison(Predicate): + field: str + operator: ComparisonOperator + value: Scalar + + +@dataclass(frozen=True) +class In(Predicate): + field: str + values: tuple[Scalar, ...] + + +@dataclass(frozen=True) +class Contains(Predicate): + field: str + value: str + + +@dataclass(frozen=True) +class IsNull(Predicate): + field: str + + +@dataclass(frozen=True) +class All(Predicate): + children: tuple[Predicate, ...] + + +@dataclass(frozen=True) +class AnyOf(Predicate): + children: tuple[Predicate, ...] + + +def compare(field: str, operator: ComparisonOperator, value: Scalar) -> Predicate: + return Comparison(field, operator, value) + + +def eq(field: str, value: Scalar) -> Predicate: + return compare(field, "eq", value) + + +def ne(field: str, value: Scalar) -> Predicate: + return compare(field, "ne", value) + + +def gt(field: str, value: Scalar) -> Predicate: + return compare(field, "gt", value) + + +def gte(field: str, value: Scalar) -> Predicate: + return compare(field, "gte", value) + + +def lt(field: str, value: Scalar) -> Predicate: + return compare(field, "lt", value) + + +def lte(field: str, value: Scalar) -> Predicate: + return compare(field, "lte", value) + + +def one_of(field: str, values: list[Scalar] | tuple[Scalar, ...]) -> Predicate: + if not values: + raise ValueError("one_of requires at least one value") + return In(field, tuple(values)) + + +def contains(field: str, value: str) -> Predicate: + return Contains(field, value) + + +def is_null(field: str) -> Predicate: + return IsNull(field) + + +def all_of(*predicates: Predicate | None) -> Predicate: + """AND the given predicates, flattening nested ``All`` and dropping ``None``. + + An empty result is rejected rather than rendered: adapters emit ``""`` for + an empty group, and an empty filter means *match every row* — a silent + "delete everything" if it ever reached :meth:`IndexRepository.delete`. + """ + children: list[Predicate] = [] + for predicate in predicates: + if predicate is None: + continue + if isinstance(predicate, All): + children.extend(predicate.children) + else: + children.append(predicate) + if not children: + raise ValueError("all_of requires at least one non-None predicate") + return All(tuple(children)) + + +def any_of(*predicates: Predicate | None) -> Predicate: + """OR the given predicates, flattening nested ``AnyOf`` and dropping ``None``. + + Empty is rejected for the same reason as :func:`all_of`, and the inversion + is worse here: an empty OR means "match nothing" but would render as + "match everything". + """ + children: list[Predicate] = [] + for predicate in predicates: + if predicate is None: + continue + if isinstance(predicate, AnyOf): + children.extend(predicate.children) + else: + children.append(predicate) + if not children: + raise ValueError("any_of requires at least one non-None predicate") + return AnyOf(tuple(children)) + + +__all__ = [ + "All", + "AnyOf", + "Comparison", + "Contains", + "In", + "IsNull", + "Predicate", + "Scalar", + "all_of", + "any_of", + "compare", + "contains", + "eq", + "gt", + "gte", + "is_null", + "lt", + "lte", + "ne", + "one_of", +] diff --git a/src/everos/memory/cascade/_backfill.py b/src/everos/memory/cascade/_backfill.py index 767ee6d24..11f2ac850 100644 --- a/src/everos/memory/cascade/_backfill.py +++ b/src/everos/memory/cascade/_backfill.py @@ -42,25 +42,30 @@ from everos.core.errors import ProviderNotConfiguredError from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, MemoryRoot, SQLModel -from everos.core.persistence.lancedb import BaseLanceTable, LanceRepoBase from everos.infra.ome.config import OMEConfig from everos.infra.ome.engine import OfflineEngine from everos.infra.ome.exceptions import EngineLockHeldError -from everos.infra.persistence.lancedb import ( - BUSINESS_SCHEMAS_WITH_VECTOR, +from everos.infra.persistence.index import ( + ALL_REPOS, AgentCase, AgentSkill, AtomicFact, Episode, Foresight, + IndexRepository, KnowledgeTopic, + Predicate, agent_case_repo, agent_skill_repo, + any_of, atomic_fact_repo, episode_repo, + eq, foresight_repo, - get_table, + is_null, knowledge_topic_repo, + repo_for_schema, + schema_for, ) from everos.infra.persistence.markdown import AgentSkillFrontmatter from everos.infra.persistence.sqlite import cluster_repo, get_engine @@ -223,8 +228,8 @@ class _TableSpec: ``subject`` column — mirrors ``EpisodeHandler._build_row``. """ - schema: type[BaseLanceTable] - repo: LanceRepoBase[Any] + schema: type[Any] + repo: IndexRepository[Any] text_of: Callable[[dict[str, Any]], str] subject_of: Callable[[dict[str, Any]], str | None] | None = None @@ -257,10 +262,14 @@ def _agent_skill_embed_text(row: dict[str, Any]) -> str: # touches. Fail loud at import time so any drift shows up before a test # even starts. _TABLE_SPEC_NAMES = {spec.schema.TABLE_NAME for spec in _TABLE_SPECS} -_BUSINESS_SCHEMA_NAMES = {s.TABLE_NAME for s in BUSINESS_SCHEMAS_WITH_VECTOR} +_BUSINESS_SCHEMA_NAMES = { + repo.schema.TABLE_NAME + for repo in ALL_REPOS + if schema_for(repo.schema).vector_fields +} if _TABLE_SPEC_NAMES != _BUSINESS_SCHEMA_NAMES: raise RuntimeError( - f"_TABLE_SPECS drift: BUSINESS_SCHEMAS_WITH_VECTOR has " + f"_TABLE_SPECS drift: the vector-carrying index repos have " f"{_BUSINESS_SCHEMA_NAMES}, _TABLE_SPECS covers " f"{_TABLE_SPEC_NAMES}. Update _TABLE_SPECS to include every " f"business table with a nullable vector column." @@ -340,12 +349,7 @@ class _PhaseResult: blocked_by_server: bool = False -def _q(value: str) -> str: - """Defensive SQL-quote escape (mirrors the lancedb repo convention).""" - return value.replace("'", "''") - - -def _null_filter(spec: _TableSpec) -> str: +def _null_filter(spec: _TableSpec) -> Predicate: """Where-clause the scan uses to spot rows that need (some) embedding. Episode carries a ``subject_vector`` alongside ``vector`` — the two @@ -362,8 +366,8 @@ def _null_filter(spec: _TableSpec) -> str: query, so they keep the round-1 filter. """ if spec.schema.TABLE_NAME == Episode.TABLE_NAME: - return "vector IS NULL OR subject_vector IS NULL" - return "vector IS NULL" + return any_of(is_null("vector"), is_null("subject_vector")) + return is_null("vector") class _RowSkipped: @@ -456,8 +460,8 @@ async def _scan_null_vector_backlog() -> tuple[list[_TableBacklog], int]: backlog: list[_TableBacklog] = [] scan_failed = 0 for spec in _TABLE_SPECS: - table = await get_table(spec.schema.TABLE_NAME, spec.schema) - raw_rows = await table.query().where(_null_filter(spec)).to_list() + records = await spec.repo.scan(_null_filter(spec)) + raw_rows = [record.model_dump(mode="python") for record in records] rows: list[_NullVectorRow] = [] for raw in raw_rows: row = _extract_row(raw, spec, tokenizer) @@ -697,7 +701,7 @@ async def _backfill_table( and row.id not in subject_vectors ) try: - await backlog.spec.repo.update(updates, where=f"id = '{_q(row.id)}'") + await backlog.spec.repo.update(updates, where=eq("id", row.id)) except Exception: result.rows_failed += 1 logger.warning( @@ -917,7 +921,7 @@ class _ClusterPhaseResult: blocked_by_capability: str | None = None -async def _scan_all_rows(schema: type[BaseLanceTable]) -> list[dict[str, Any]]: +async def _scan_all_rows(schema: type[Any]) -> list[dict[str, Any]]: """Fetch every row of ``schema``'s table, no filter. Phase 2 doesn't care whether a row already carries a vector — the @@ -934,11 +938,8 @@ async def _scan_all_rows(schema: type[BaseLanceTable]) -> list[dict[str, Any]]: apply it themselves on the returned rows — this helper stays a plain unfiltered fetch shared by every business table. """ - table = await get_table(schema.TABLE_NAME, schema) - total = await table.count_rows() - if total == 0: - return [] - return await table.query().limit(total).to_list() + records = await repo_for_schema(schema).scan() + return [record.model_dump(mode="python") for record in records] def _episode_row_to_event(raw: dict[str, Any]) -> EpisodeExtracted: diff --git a/src/everos/memory/cascade/handlers/_daily_log_base.py b/src/everos/memory/cascade/handlers/_daily_log_base.py index 8791a418f..dc59287c5 100644 --- a/src/everos/memory/cascade/handlers/_daily_log_base.py +++ b/src/everos/memory/cascade/handlers/_daily_log_base.py @@ -15,7 +15,7 @@ — editing them does NOT propagate to LanceDB and does NOT waste an embed call. -Subclasses bind their ``kind`` / ``lance_repo`` / ``content_change_keys`` +Subclasses bind their ``kind`` / ``index_repo`` / ``content_change_keys`` as ClassVars and override :meth:`_build_row` to do the per-kind field mapping. Everything else — read, diff, embed call, upsert, delete — lives here. @@ -30,6 +30,7 @@ from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, StructuredEntry +from everos.infra.persistence.index import all_of, eq, one_of from ..types import HandlerOutcome from ._common import content_sha256 as compute_content_sha256 @@ -57,7 +58,7 @@ class BaseDailyLogHandler(Handler): Subclass requirements: - :attr:`kind` (ClassVar[str]) — registry name, surfaces in logs. - - :attr:`lance_repo` (ClassVar) — the LanceDB repo singleton for + - :attr:`index_repo` (ClassVar) — the LanceDB repo singleton for this kind (must expose ``find_where`` / ``upsert`` / ``delete`` / ``delete_by_md_path``). - :attr:`content_change_keys` (ClassVar[tuple[str, ...]]) — the @@ -70,7 +71,7 @@ class BaseDailyLogHandler(Handler): """ kind: ClassVar[str] = "" - lance_repo: ClassVar[Any] = None + index_repo: ClassVar[Any] = None content_change_keys: ClassVar[tuple[str, ...]] = () def _content_sha256(self, structured: StructuredEntry) -> str: @@ -107,8 +108,8 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: for entry in parsed.entries ] - existing = await self.lance_repo.find_where( - f"md_path = '{_q(md_path)}'", + existing = await self.index_repo.find_where( + eq("md_path", md_path), limit=10_000, ) owner_id, owner_type = resolve_owner(parsed.frontmatter, md_path) @@ -196,15 +197,14 @@ async def _apply_lance_changes( ) -> None: """Flush upserts and deletes to LanceDB.""" if to_upsert: - await self.lance_repo.upsert(to_upsert) + await self.index_repo.upsert(to_upsert) if to_delete_ids: - in_list = ", ".join(f"'{eid}'" for eid in to_delete_ids) - await self.lance_repo.delete( - f"md_path = '{_q(md_path)}' AND entry_id IN ({in_list})" + await self.index_repo.delete( + all_of(eq("md_path", md_path), one_of("entry_id", to_delete_ids)) ) async def handle_deleted(self, md_path: str) -> HandlerOutcome: - deleted = await self.lance_repo.delete_by_md_path(md_path) + deleted = await self.index_repo.delete_by_md_path(md_path) return HandlerOutcome( md_path=md_path, kind=self.kind, @@ -252,14 +252,14 @@ async def _mark_deprecated( entry may have been deleted or not yet indexed. """ app_id, project_id = scope - predicate = ( - f"owner_id = '{_q(owner_id)}' " - f"AND entry_id = '{_q(entry_id)}' " - f"AND app_id = '{_q(app_id)}' " - f"AND project_id = '{_q(project_id)}'" + predicate = all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), ) try: - await self.lance_repo.update( + await self.index_repo.update( {"deprecated_by": deprecated_by}, where=predicate, ) @@ -290,8 +290,3 @@ async def _build_row( ``"default"`` so white-box callers exercising only the field mapping can omit them. """ - - -def _q(text: str) -> str: - """Defensive SQL-quote escape (mirrors lancedb chassis convention).""" - return text.replace("'", "''") diff --git a/src/everos/memory/cascade/handlers/agent_case.py b/src/everos/memory/cascade/handlers/agent_case.py index a6fa9c5b7..5ba0fae3b 100644 --- a/src/everos/memory/cascade/handlers/agent_case.py +++ b/src/everos/memory/cascade/handlers/agent_case.py @@ -30,7 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import AgentCase, ParentType, agent_case_repo +from everos.infra.persistence.index import AgentCase, ParentType, agent_case_repo from ._common import require_float, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry @@ -42,7 +42,7 @@ class AgentCaseHandler(BaseDailyLogHandler): """Cascade handler for ``agents//.cases/agent_case-*.md``.""" kind = "agent_case" - lance_repo = agent_case_repo + index_repo = agent_case_repo content_change_keys = ( "section:TaskIntent", "section:Approach", diff --git a/src/everos/memory/cascade/handlers/agent_skill.py b/src/everos/memory/cascade/handlers/agent_skill.py index 88ae83959..a72d1371b 100644 --- a/src/everos/memory/cascade/handlers/agent_skill.py +++ b/src/everos/memory/cascade/handlers/agent_skill.py @@ -42,7 +42,13 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import AgentSkill, agent_skill_repo +from everos.infra.persistence.index import ( + AgentSkill, + agent_skill_repo, + all_of, + eq, + ne, +) from everos.infra.persistence.markdown import AgentSkillFrontmatter from ..types import HandlerOutcome @@ -58,7 +64,7 @@ class AgentSkillHandler(Handler): ``agents//skills/skill_/SKILL.md`` + ``references/*.md``.""" kind = "agent_skill" - lance_repo: ClassVar[Any] = agent_skill_repo + index_repo: ClassVar[Any] = agent_skill_repo """Exposed for ``CascadeWorker._optimize_touched_kinds`` — see the matching note on :class:`UserProfileHandler`.""" @@ -136,7 +142,7 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: # cascade's contract — skip the lookup. deleted = 0 if prior is None: - orphan_clause = f"md_path = '{_q(md_path)}' AND id != '{_q(skill_id)}'" + orphan_clause = all_of(eq("md_path", md_path), ne("id", skill_id)) orphans = await agent_skill_repo.find_where(orphan_clause, limit=1000) deleted = len(orphans) if deleted: @@ -224,8 +230,3 @@ def _join_body_and_references(body: str, references: str) -> str: if not body: return references return f"{body}\n\n{references}" - - -def _q(value: str) -> str: - """Defensive SQL-quote escape (mirrors lancedb chassis convention).""" - return value.replace("'", "''") diff --git a/src/everos/memory/cascade/handlers/atomic_fact.py b/src/everos/memory/cascade/handlers/atomic_fact.py index 6dea35e4e..6dee3b94a 100644 --- a/src/everos/memory/cascade/handlers/atomic_fact.py +++ b/src/everos/memory/cascade/handlers/atomic_fact.py @@ -23,7 +23,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import AtomicFact, ParentType, atomic_fact_repo +from everos.infra.persistence.index import AtomicFact, ParentType, atomic_fact_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry @@ -35,7 +35,7 @@ class AtomicFactHandler(BaseDailyLogHandler): """Cascade handler for ``users//.atomic_facts/atomic_fact-*.md``.""" kind = "atomic_fact" - lance_repo = atomic_fact_repo + index_repo = atomic_fact_repo content_change_keys = ("section:Fact",) """Only ``Fact`` matters — it's both the embedded text AND the BM25 source. Audit inline is excluded.""" diff --git a/src/everos/memory/cascade/handlers/episode.py b/src/everos/memory/cascade/handlers/episode.py index 84878d302..c8980650b 100644 --- a/src/everos/memory/cascade/handlers/episode.py +++ b/src/everos/memory/cascade/handlers/episode.py @@ -38,7 +38,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import Episode, ParentType, episode_repo +from everos.infra.persistence.index import Episode, ParentType, episode_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry @@ -50,7 +50,7 @@ class EpisodeHandler(BaseDailyLogHandler): """Cascade handler for ``users//episodes/episode-*.md``.""" kind = "episode" - lance_repo = episode_repo + index_repo = episode_repo content_change_keys = ( "section:Subject", "section:Summary", diff --git a/src/everos/memory/cascade/handlers/foresight.py b/src/everos/memory/cascade/handlers/foresight.py index adbefe020..2c618df18 100644 --- a/src/everos/memory/cascade/handlers/foresight.py +++ b/src/everos/memory/cascade/handlers/foresight.py @@ -30,7 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import Foresight, ParentType, foresight_repo +from everos.infra.persistence.index import Foresight, ParentType, foresight_repo from ._common import ( optional_int, @@ -47,7 +47,7 @@ class ForesightHandler(BaseDailyLogHandler): """Cascade handler for ``users//.foresights/foresight-*.md``.""" kind = "foresight" - lance_repo = foresight_repo + index_repo = foresight_repo content_change_keys = ( "section:Foresight", "section:Evidence", diff --git a/src/everos/memory/cascade/handlers/knowledge_topic.py b/src/everos/memory/cascade/handlers/knowledge_topic.py index daa8fbc3c..16fc443a2 100644 --- a/src/everos/memory/cascade/handlers/knowledge_topic.py +++ b/src/everos/memory/cascade/handlers/knowledge_topic.py @@ -38,7 +38,7 @@ from everos.component.utils.datetime import get_utc_now from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, ParsedMarkdown -from everos.infra.persistence.lancedb import KnowledgeTopic, knowledge_topic_repo +from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo from everos.infra.persistence.sqlite import ( TopicUpsertPayload, knowledge_topic_sqlite_repo, @@ -57,7 +57,7 @@ class KnowledgeTopicHandler(Handler): ``knowledge/{category}/{doc_title}/_.md``.""" kind = "knowledge_topic" - lance_repo: ClassVar[Any] = knowledge_topic_repo + index_repo: ClassVar[Any] = knowledge_topic_repo content_change_keys: ClassVar[tuple[str, ...]] = ( "frontmatter:summary", diff --git a/src/everos/memory/cascade/handlers/user_profile.py b/src/everos/memory/cascade/handlers/user_profile.py index 4842986d0..460a017d6 100644 --- a/src/everos/memory/cascade/handlers/user_profile.py +++ b/src/everos/memory/cascade/handlers/user_profile.py @@ -27,7 +27,7 @@ from typing import Any, ClassVar from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import UserProfile, user_profile_repo +from everos.infra.persistence.index import UserProfile, user_profile_repo from ..types import HandlerOutcome from ._common import content_sha256 as compute_content_sha256 @@ -39,7 +39,7 @@ class UserProfileHandler(Handler): """Cascade handler for ``users//user.md``.""" kind = "user_profile" - lance_repo: ClassVar[Any] = user_profile_repo + index_repo: ClassVar[Any] = user_profile_repo """Exposed for ``CascadeWorker._optimize_touched_kinds`` — the worker discovers the LanceDB repo to optimize via this attribute, mirroring the daily-log handlers that bind it through diff --git a/src/everos/memory/cascade/registry.py b/src/everos/memory/cascade/registry.py index 8537575e6..feecebbe3 100644 --- a/src/everos/memory/cascade/registry.py +++ b/src/everos/memory/cascade/registry.py @@ -18,7 +18,7 @@ from pathlib import PurePosixPath from everos.core.persistence.markdown import BaseFrontmatter -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase, AgentSkill, AtomicFact, @@ -66,7 +66,7 @@ class KindSpec: ``frontmatter_schema`` carries the ``path_glob()`` classmethod the scanner uses to enumerate eligible files; the same schema is also the contract the reader / writer share at the markdown layer. - ``lance_schema`` + ``lance_repo`` describe the destination side. + ``index_schema`` + ``index_repo`` describe the destination side. ``handler_factory`` is a callable that receives the shared :class:`HandlerDeps` bundle and returns the kind's :class:`Handler`. """ @@ -74,8 +74,8 @@ class KindSpec: name: str frontmatter_schema: type[BaseFrontmatter] handler_factory: type[Handler] - lance_schema: type | None = None - lance_repo: object | None = None + index_schema: type | None = None + index_repo: object | None = None def path_glob(self) -> str: """Glob (relative to memory root) for every md this kind covers.""" @@ -96,58 +96,58 @@ def matches(self, rel_md_path: str) -> bool: KindSpec( name="episode", frontmatter_schema=EpisodeDailyFrontmatter, - lance_schema=Episode, - lance_repo=episode_repo, + index_schema=Episode, + index_repo=episode_repo, handler_factory=EpisodeHandler, ), KindSpec( name="atomic_fact", frontmatter_schema=AtomicFactDailyFrontmatter, - lance_schema=AtomicFact, - lance_repo=atomic_fact_repo, + index_schema=AtomicFact, + index_repo=atomic_fact_repo, handler_factory=AtomicFactHandler, ), KindSpec( name="foresight", frontmatter_schema=ForesightDailyFrontmatter, - lance_schema=Foresight, - lance_repo=foresight_repo, + index_schema=Foresight, + index_repo=foresight_repo, handler_factory=ForesightHandler, ), KindSpec( name="agent_case", frontmatter_schema=AgentCaseDailyFrontmatter, - lance_schema=AgentCase, - lance_repo=agent_case_repo, + index_schema=AgentCase, + index_repo=agent_case_repo, handler_factory=AgentCaseHandler, ), KindSpec( name="agent_skill", frontmatter_schema=AgentSkillFrontmatter, - lance_schema=AgentSkill, - lance_repo=agent_skill_repo, + index_schema=AgentSkill, + index_repo=agent_skill_repo, handler_factory=AgentSkillHandler, ), KindSpec( name="user_profile", frontmatter_schema=UserProfileFrontmatter, - lance_schema=UserProfile, - lance_repo=user_profile_repo, + index_schema=UserProfile, + index_repo=user_profile_repo, handler_factory=UserProfileHandler, ), KindSpec( name="knowledge_document", frontmatter_schema=KnowledgeDocumentFrontmatter, handler_factory=KnowledgeDocumentHandler, - lance_schema=None, - lance_repo=None, + index_schema=None, + index_repo=None, ), KindSpec( name="knowledge_topic", frontmatter_schema=KnowledgeTopicFrontmatter, handler_factory=KnowledgeTopicHandler, - lance_schema=KnowledgeTopic, - lance_repo=knowledge_topic_repo, + index_schema=KnowledgeTopic, + index_repo=knowledge_topic_repo, ), ) """Every cascade kind, evaluated in declaration order by :func:`match_kind`.""" diff --git a/src/everos/memory/cascade/worker.py b/src/everos/memory/cascade/worker.py index 42232469a..7ab050a05 100644 --- a/src/everos/memory/cascade/worker.py +++ b/src/everos/memory/cascade/worker.py @@ -879,7 +879,7 @@ def _schedule_optimize(self, kind: str) -> None: LanceDB's per-table manifest version expects. No-op when the handler for ``kind`` doesn't expose a - ``lance_repo`` (test stubs, handlers that intentionally skip + ``index_repo`` (test stubs, handlers that intentionally skip LanceDB). Idempotent and cheap (a single dict lookup + flag write in @@ -887,7 +887,7 @@ def _schedule_optimize(self, kind: str) -> None: heartbeat sweep. """ handler = self._handlers.get(kind) - repo = getattr(handler, "lance_repo", None) if handler else None + repo = getattr(handler, "index_repo", None) if handler else None if repo is None: return state = self._optimizer_states.setdefault(kind, _KindOptimizerState()) @@ -987,7 +987,7 @@ async def _run_optimize_once(self, kind: str) -> None: manifest, and prune is a write. """ handler = self._handlers.get(kind) - repo = getattr(handler, "lance_repo", None) if handler else None + repo = getattr(handler, "index_repo", None) if handler else None if repo is None: return state = self._optimizer_states.get(kind) @@ -1201,7 +1201,7 @@ async def _run_rebuild_once(self, kind: str) -> None: during the rebuild coalesce instead of racing on the manifest. """ handler = self._handlers.get(kind) - repo = getattr(handler, "lance_repo", None) if handler else None + repo = getattr(handler, "index_repo", None) if handler else None if repo is None: return state = self._optimizer_states.setdefault(kind, _KindOptimizerState()) diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index 407a06bc7..63dd46846 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -15,6 +15,7 @@ from __future__ import annotations +from everos.infra.persistence.index import Predicate from everos.memory.search import FilterNode, compile_filters @@ -25,7 +26,7 @@ def compile_filters_for_get( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> str: +) -> Predicate: """Compile ``/get`` filters via the shared ``compile_filters`` path. Kept as a named wrapper so ``memory.get`` consumers depend on a diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index c42edd7ea..8270323cc 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -37,11 +37,11 @@ from .filters_adapter import compile_filters_for_get if TYPE_CHECKING: - from everos.core.persistence.lancedb import LanceRepoBase - from everos.infra.persistence.lancedb import ( + from everos.infra.persistence.index import ( AgentCase, AgentSkill, Episode, + IndexRepository, UserProfile, ) @@ -55,10 +55,10 @@ class GetManager: def __init__( self, *, - episode_repo: LanceRepoBase[Episode], - agent_case_repo: LanceRepoBase[AgentCase], - agent_skill_repo: LanceRepoBase[AgentSkill], - user_profile_repo: LanceRepoBase[UserProfile], + episode_repo: IndexRepository[Episode], + agent_case_repo: IndexRepository[AgentCase], + agent_skill_repo: IndexRepository[AgentSkill], + user_profile_repo: IndexRepository[UserProfile], ) -> None: self._ep = episode_repo self._case = agent_case_repo diff --git a/src/everos/memory/reflection/orchestrator.py b/src/everos/memory/reflection/orchestrator.py index 547adc643..58b04d50c 100644 --- a/src/everos/memory/reflection/orchestrator.py +++ b/src/everos/memory/reflection/orchestrator.py @@ -32,6 +32,7 @@ from everos.core.observability.tracing import memory_span from everos.core.persistence import MemoryRoot from everos.infra.ome.context import StrategyContext +from everos.infra.persistence.index import all_of, eq, is_null from everos.memory._partition_locks import get_partition_lock from everos.memory.events import EpisodeExtracted @@ -41,21 +42,6 @@ _WAIT_TIMEOUT_SECONDS = 120.0 -def _escape_sql(value: str) -> str: - """Escape single quotes for LanceDB SQL-like ``where`` predicates. - - LanceDB has no parameterised query API; doubling the quote - (``'`` -> ``''``) is the SQL-standard escape. - - Args: - value: Raw string to escape. - - Returns: - Escaped string safe for interpolation into a WHERE clause. - """ - return value.replace("'", "''") - - class ReflectionOrchestrator: """Run one Reflection cycle for a single owner scope. @@ -531,12 +517,13 @@ async def _detect_orphans( app_id: Application scope. project_id: Project scope. """ - where = ( - f"parent_type = 'cluster' AND parent_id = '{_escape_sql(cluster_id)}' " - f"AND deprecated_by IS NULL " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND app_id = '{_escape_sql(app_id)}' " - f"AND project_id = '{_escape_sql(project_id)}'" + where = all_of( + eq("parent_type", "cluster"), + eq("parent_id", cluster_id), + is_null("deprecated_by"), + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), ) orphans = await self._episode_store.find_where(where, limit=10) if orphans: @@ -865,11 +852,11 @@ async def _deprecate_lance_episodes( coros: list[Any] = [ self._episode_store.update( {"deprecated_by": merged_entry_id}, - where=( - f"entry_id = '{_escape_sql(eid)}' " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND app_id = '{_escape_sql(app_id)}' " - f"AND project_id = '{_escape_sql(project_id)}'" + where=all_of( + eq("entry_id", eid), + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), ), ) for eid in entry_ids @@ -901,10 +888,10 @@ async def _deprecate_lance_facts( coros = [ self._atomic_fact_store.update( {"deprecated_by": merged_entry_id}, - where=( - f"parent_id = '{_escape_sql(pid)}' " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND deprecated_by IS NULL" + where=all_of( + eq("parent_id", pid), + eq("owner_id", owner_id), + is_null("deprecated_by"), ), ) for pid in parent_ids diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 2dd9aa804..927a07052 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -1,68 +1,38 @@ -"""Filters DSL → LanceDB ``where`` string compiler. - -The Filters DSL is intentionally permissive at the JSON layer (so callers -can pass whatever they like and get a clean 400 if it is not supported) -and rigid at compile time. Field names are validated against a small -allow-list; operators against a closed enum; string literals are -single-quote-escaped. Timestamps are accepted as epoch milliseconds and -rendered as DataFusion ``TIMESTAMP ''`` literals. - -``owner_id`` and ``owner_type`` are the hard partition keys; they are -not part of the DSL at all. :func:`compile_filters` injects them at the -top of the compiled string from :class:`SearchRequest` and rejects any -attempt to override them inside ``filters``. - -Public surface --------------- - -The compiler exposes three primitives so adjacent subpackages -(notably ``memory.get``) can build narrower DSLs without forking the -field allow-list: - -* :data:`ALLOWED_FIELDS` — mapping ``field_name → _FieldSpec`` (column + - kind). Iterate / membership-test only; do not mutate. -* :data:`RESERVED_FIELDS` — names rejected inside any ``filters`` block. -* :func:`compile_predicate` — render one ``{field: value}`` clause to - SQL. Operator-map and equality-shorthand are both handled. - -The high-level :func:`compile_filters` remains the entry point for -``/search`` (combinator-aware). +"""Validate the public filter DSL and build a backend-neutral predicate. + +Storage-specific syntax is intentionally absent from this module. Adapters +render the resulting predicate in their own persistence packages. """ from __future__ import annotations -import datetime as _dt -from typing import Any, Final +import datetime as dt +from dataclasses import dataclass +from typing import Any, Final, Literal -from everos.component.utils.datetime import from_timestamp, to_iso_format +from everos.component.utils.datetime import ensure_utc, from_iso_format, from_timestamp from everos.core.errors import FilterError as FilterError +from everos.infra.persistence.index import ( + Predicate, + Scalar, + all_of, + any_of, + compare, + contains, + eq, + is_null, + one_of, +) from .dto import FilterNode -# ── Allow-lists ────────────────────────────────────────────────────────── - -_OP_MAP: Final[dict[str, str]] = { - "eq": "=", - "ne": "!=", - "gt": ">", - "gte": ">=", - "lt": "<", - "lte": "<=", - "in": "IN", -} - -# Field kinds: ``str`` rendered as ``''``; ``ts`` rendered as -# ``TIMESTAMP ''`` (DataFusion timestamp literal); ``array_str`` -# uses DataFusion's ``array_has`` on a list column. -_FieldKind = str # one of: "str" | "ts" | "array_str" +_FieldKind = Literal["str", "ts", "array_str"] +@dataclass(frozen=True) class _FieldSpec: - __slots__ = ("column", "kind") - - def __init__(self, column: str, kind: _FieldKind) -> None: - self.column = column - self.kind = kind + column: str + kind: _FieldKind ALLOWED_FIELDS: Final[dict[str, _FieldSpec]] = { @@ -73,18 +43,11 @@ def __init__(self, column: str, kind: _FieldKind) -> None: "sender_id": _FieldSpec("sender_ids", "array_str"), } -# Fields the caller is explicitly **not** allowed to place inside -# ``filters``; they live at the top of :class:`SearchRequest` and are -# injected by :func:`compile_filters`. Rejecting them here turns a -# silent override into a 400. RESERVED_FIELDS: Final[frozenset[str]] = frozenset( {"owner_id", "owner_type", "app_id", "project_id"} ) -# ── Public API ─────────────────────────────────────────────────────────── - - def compile_filters( node: FilterNode | None, *, @@ -92,50 +55,35 @@ def compile_filters( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> str: - """Compile a request's filters into a single LanceDB ``where`` string. - - The base clause always pins the hard partition keys (``owner_id`` / - ``owner_type`` and the ``app_id`` / ``project_id`` scope segments) to - the request's top-level values; anything in ``node`` is appended with - an ``AND``. Pinning app/project here is what isolates one space's rows - from another — omitting it would let a query bleed across spaces. Both - ``/search`` and ``/get`` share this compile path. - """ - base = [ - f"owner_id = '{_escape_str(owner_id)}'", - f"owner_type = '{owner_type}'", - f"app_id = '{_escape_str(app_id)}'", - f"project_id = '{_escape_str(project_id)}'", +) -> Predicate: + """Validate request filters and return one normalized predicate tree.""" + base: list[Predicate] = [ + eq("owner_id", owner_id), + eq("owner_type", owner_type), + eq("app_id", app_id), + eq("project_id", project_id), ] - # Only episode / atomic_fact tables carry the ``deprecated_by`` column - # (Reflection V1 marks superseded entries). Agent tables don't have it. if owner_type == "user": - base.append("deprecated_by IS NULL") - if node is None: - return " AND ".join(base) - compiled = _compile_node(node.model_dump(exclude_none=True)) - if not compiled: - return " AND ".join(base) - return " AND ".join([*base, compiled]) - - -# ── Internals ──────────────────────────────────────────────────────────── + base.append(is_null("deprecated_by")) + if node is not None: + compiled = _compile_node(node.model_dump(exclude_none=True)) + if compiled is not None: + base.append(compiled) + return all_of(*base) -def _compile_node(raw: dict[str, Any]) -> str: - """Walk one DSL node; return the matching SQL fragment (no leading parens). - - Empty nodes yield ``""`` so :func:`compile_filters` can skip the - trailing ``AND``. - """ - raw = dict(raw) # never mutate the caller's dict - parts: list[str] = [] +def _compile_node(raw: dict[str, Any]) -> Predicate | None: + raw = dict(raw) + parts: list[Predicate] = [] if (and_list := raw.pop("AND", None)) is not None: - parts.append(_compile_combinator(and_list, "AND")) + combinator = _compile_combinator(and_list, "AND") + if combinator is not None: + parts.append(combinator) if (or_list := raw.pop("OR", None)) is not None: - parts.append(_compile_combinator(or_list, "OR")) + combinator = _compile_combinator(or_list, "OR") + if combinator is not None: + parts.append(combinator) for field, value in raw.items(): if field in RESERVED_FIELDS: @@ -146,122 +94,99 @@ def _compile_node(raw: dict[str, Any]) -> str: raise FilterError(f"unsupported filter field: {field!r}") parts.append(compile_predicate(field, value)) - # Drop empty fragments coming from empty AND/OR arrays. - parts = [p for p in parts if p] - if not parts: - return "" - if len(parts) == 1: - return parts[0] - return " AND ".join(parts) + return all_of(*parts) if parts else None -def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: - """Render an ``AND`` / ``OR`` array of child nodes.""" +def _compile_combinator( + children: list[dict[str, Any]], op: Literal["AND", "OR"] +) -> Predicate | None: if not isinstance(children, list): raise FilterError(f"{op} expects an array of nodes") - fragments: list[str] = [] + fragments: list[Predicate] = [] for child in children: if not isinstance(child, dict): raise FilterError(f"{op} children must be objects") compiled = _compile_node(child) - if compiled: - fragments.append(f"({compiled})") + if compiled is not None: + fragments.append(compiled) if not fragments: - return "" - if len(fragments) == 1: - # No need for the surrounding combinator when only one effective child. - return fragments[0] - glue = f" {op} " - return "(" + glue.join(fragments) + ")" - - -def compile_predicate(field: str, value: Any) -> str: - """Render one ``"": `` clause to SQL. + return None + return all_of(*fragments) if op == "AND" else any_of(*fragments) - Public primitive — :mod:`memory.get` builds a flat (no AND/OR) - DSL on top of it. Callers must pre-validate ``field`` against - :data:`ALLOWED_FIELDS` and :data:`RESERVED_FIELDS`; this function - will ``KeyError`` on unknown fields. - ``value`` is either a scalar (equality shorthand) or an - ``{"": }`` map. Mixing multiple operators in one - dict is allowed and folds with ``AND``:: - - "timestamp": {"gte": 1, "lt": 2} - → (timestamp >= TIMESTAMP '...' AND timestamp < TIMESTAMP '...') - """ +def compile_predicate(field: str, value: Any) -> Predicate: + """Validate and normalize one field clause into the neutral AST.""" spec = ALLOWED_FIELDS[field] if isinstance(value, dict): if not value: raise FilterError(f"empty operator map for field {field!r}") - clauses = [ - _compile_op_clause(spec, field, op, op_val) for op, op_val in value.items() - ] - if len(clauses) == 1: - return clauses[0] - return "(" + " AND ".join(clauses) + ")" - # Equality shorthand. + return all_of( + *( + _compile_op_clause(spec, field, op, op_value) + for op, op_value in value.items() + ) + ) return _compile_op_clause(spec, field, "eq", value) -def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> str: - """Render a single `` `` clause.""" - if op not in _OP_MAP: +def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> Predicate: + if op not in {"eq", "ne", "gt", "gte", "lt", "lte", "in"}: raise FilterError(f"unsupported operator {op!r} on field {field!r}") - sql_op = _OP_MAP[op] if spec.kind == "array_str": - # Only equality / membership make sense on a list column. if op == "eq": - literal = _escape_str(_require_str(value, field)) - return f"array_has({spec.column}, '{literal}')" + return contains(spec.column, _require_str(value, field)) if op == "in": - items = _require_list(value, field) - literals = [f"'{_escape_str(_require_str(v, field))}'" for v in items] - inner = " OR ".join(f"array_has({spec.column}, {lit})" for lit in literals) - return f"({inner})" + values = _require_list(value, field) + return any_of( + *(contains(spec.column, _require_str(item, field)) for item in values) + ) raise FilterError(f"operator {op!r} is not supported on array field {field!r}") if op == "in": - items = _require_list(value, field) - literals = [_render_literal(v, spec.kind, field) for v in items] - return f"{spec.column} IN ({', '.join(literals)})" - - return f"{spec.column} {sql_op} {_render_literal(value, spec.kind, field)}" - - -# ── Literal rendering ──────────────────────────────────────────────────── - - -def _render_literal(value: Any, kind: _FieldKind, field: str) -> str: + values = _require_list(value, field) + return one_of( + spec.column, + [_normalize_literal(item, spec.kind, field) for item in values], + ) + return compare( + spec.column, + op, # type: ignore[arg-type] + _normalize_literal(value, spec.kind, field), + ) + + +def _normalize_literal(value: Any, kind: _FieldKind, field: str) -> Scalar: if kind == "str": - return f"'{_escape_str(_require_str(value, field))}'" + return _require_str(value, field) if kind == "ts": - return f"TIMESTAMP '{_render_ts(value, field)}'" + return _normalize_timestamp(value, field) raise FilterError(f"unsupported field kind {kind!r} for field {field!r}") -def _render_ts(value: Any, field: str) -> str: - """Accept epoch ms (int / float) or an ISO 8601 string; emit ISO.""" - if isinstance(value, bool): # bools subclass int — reject early +def _normalize_timestamp(value: Any, field: str) -> dt.datetime: + if isinstance(value, bool): raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") - if isinstance(value, (int, float)): - return to_iso_format(from_timestamp(int(value))) - if isinstance(value, str): - # Trust the caller-supplied ISO string but escape quotes defensively. - if "'" in value: - raise FilterError(f"timestamp string for {field!r} contains a quote") - return value - if isinstance(value, _dt.datetime): - return to_iso_format(value) + try: + if isinstance(value, (int, float)): + return from_timestamp(int(value)) + if isinstance(value, str): + if "'" in value: + raise FilterError(f"timestamp string for {field!r} contains a quote") + parsed = ensure_utc(from_iso_format(value)) + assert parsed is not None + return parsed + if isinstance(value, dt.datetime): + parsed = ensure_utc(value) + assert parsed is not None + return parsed + except (TypeError, ValueError) as exc: + raise FilterError( + f"timestamp value for {field!r} must be ms or ISO string" + ) from exc raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") -def _escape_str(value: str) -> str: - """Double single quotes — SQL-standard escape for a single-quoted literal.""" - return value.replace("'", "''") - - def _require_str(value: Any, field: str) -> str: if not isinstance(value, str): raise FilterError(f"value for {field!r} must be a string") @@ -272,3 +197,11 @@ def _require_list(value: Any, field: str) -> list[Any]: if not isinstance(value, list) or not value: raise FilterError(f"value for {field!r} with 'in' must be a non-empty list") return value + + +__all__ = [ + "ALLOWED_FIELDS", + "RESERVED_FIELDS", + "compile_filters", + "compile_predicate", +] diff --git a/src/everos/memory/search/recall/agent_case.py b/src/everos/memory/search/recall/agent_case.py index 93a7f9b60..bfa71f47f 100644 --- a/src/everos/memory/search/recall/agent_case.py +++ b/src/everos/memory/search/recall/agent_case.py @@ -13,17 +13,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentCase, get_table +from everos.infra.persistence.index import AgentCase, Predicate, agent_case_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -40,7 +38,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Dual-column BM25 recall via OR-mode BooleanQuery per column. @@ -50,62 +48,23 @@ async def sparse_recall( per BM25 column (``MatchQuery`` is column-bound), then the two per-column result lists merge by id keeping the max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentCase.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentCase.BM25_FIELDS), + merged_rows = await agent_case_repo.sparse_search( + terms, where, columns=AgentCase.BM25_FIELDS, limit=limit ) - # Merge by id, keep the max BM25 score across the two columns. - # task_intent hits typically score higher (the retrieval anchor); - # approach hits catch queries that match a step detail. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_case_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/agent_skill.py b/src/everos/memory/search/recall/agent_skill.py index f444e4a38..c2ad6295d 100644 --- a/src/everos/memory/search/recall/agent_skill.py +++ b/src/everos/memory/search/recall/agent_skill.py @@ -9,26 +9,27 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentSkill, get_table +from everos.infra.persistence.index import ( + AgentSkill, + Predicate, + agent_skill_repo, + all_of, + any_of, + contains, +) from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) -def _q(value: str) -> str: - return value.replace("'", "''") - - class AgentSkillRecaller: """BM25 + vector recall over the LanceDB ``agent_skill`` table.""" @@ -40,7 +41,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Dual-column BM25 recall via OR-mode BooleanQuery per column. @@ -48,60 +49,23 @@ async def sparse_recall( rationale. One BooleanQuery per BM25 column; merge by id with max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentSkill.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentSkill.BM25_FIELDS), + merged_rows = await agent_skill_repo.sparse_search( + terms, where, columns=AgentSkill.BM25_FIELDS, limit=limit ) - # Merge by id, keep max BM25 score across the two columns. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_skill_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -112,7 +76,7 @@ async def dense_recall( ] async def fetch_by_case_ids( - self, case_ids: Sequence[str], where: str, *, limit: int + self, case_ids: Sequence[str], where: Predicate, *, limit: int ) -> list[Candidate]: """Skills whose ``source_case_ids`` intersect ``case_ids``. Filter is ``array_has`` OR-ed per id (same as @@ -124,8 +88,9 @@ async def fetch_by_case_ids( """ if not case_ids: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - clause = " OR ".join(f"array_has(source_case_ids, '{_q(c)}')" for c in case_ids) - full_where = f"({where}) AND ({clause})" - rows = await table.query().where(full_where).limit(limit).to_list() + full_where = all_of( + where, + any_of(*(contains("source_case_ids", case_id) for case_id in case_ids)), + ) + rows = await agent_skill_repo.search(where=full_where, limit=limit) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 69eba22d4..6b65f52cc 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -21,11 +21,16 @@ from everalgo.types import Candidate, FactCandidate -from everos.infra.persistence.lancedb import AtomicFact, get_table +from everos.infra.persistence.index import ( + AtomicFact, + Predicate, + all_of, + atomic_fact_repo, + one_of, +) from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -46,17 +51,14 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery (see EpisodeRecaller docstring).""" - bq = build_or_query( - self._deps.tokenizer, query, column=AtomicFact.BM25_FIELDS[0] - ) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await atomic_fact_repo.sparse_search( + terms, where, columns=AtomicFact.BM25_FIELDS, limit=limit ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -64,7 +66,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Cosine ANN recall over the atomic_fact table. @@ -78,15 +80,7 @@ async def dense_recall( """ if not vector: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await atomic_fact_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -99,7 +93,7 @@ async def dense_recall( async def facts_for_episodes( self, ep_to_parents: Mapping[str, Sequence[str]], - where: str, + where: Predicate, *, per_episode: int, query_vector: Sequence[float] | None = None, @@ -160,27 +154,19 @@ async def facts_for_episodes( async def _query_facts_for_parents( self, parent_to_eps: dict[str, list[str]], - where: str, + where: Predicate, *, per_episode: int, query_vector: Sequence[float] | None, ) -> list[dict[str, Any]]: """Construct and execute the LanceDB query for parent_id IN (...).""" - quoted = ", ".join(f"'{_q(pid)}'" for pid in parent_to_eps) - clause = f"parent_id IN ({quoted})" - full_where = f"({where}) AND ({clause})" + full_where = all_of(where, one_of("parent_id", list(parent_to_eps))) limit = per_episode * max(len(parent_to_eps), 1) - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) if query_vector: - return await ( - table.query() - .nearest_to(list(query_vector)) - .distance_type("cosine") - .where(full_where) - .limit(limit) - .to_list() + return await atomic_fact_repo.dense_search( + query_vector, full_where, limit=limit ) - return await table.query().where(full_where).limit(limit).to_list() + return await atomic_fact_repo.search(where=full_where, limit=limit) def _build_parent_to_episode_map( @@ -193,7 +179,3 @@ def _build_parent_to_episode_map( if pid: parent_to_eps[pid].append(ep_id) return parent_to_eps - - -def _q(value: str) -> str: - return value.replace("'", "''") diff --git a/src/everos/memory/search/recall/base.py b/src/everos/memory/search/recall/base.py index 9b62c10f9..e77fd2354 100644 --- a/src/everos/memory/search/recall/base.py +++ b/src/everos/memory/search/recall/base.py @@ -24,14 +24,9 @@ from typing import Any, ClassVar, Protocol, runtime_checkable from everalgo.types import Candidate -from lancedb.query import BooleanQuery, FullTextQuery, MatchQuery - -try: - from lancedb.query import Occur -except ImportError: # pragma: no cover — fallback for older LanceDB layouts - from lancedb._lancedb import Occur # type: ignore[attr-defined,no-redef] from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.index import Predicate # Columns that should never travel through the ranker / shaper. ``vector`` # is huge (1024 floats); ``_distance`` belongs to LanceDB's query engine @@ -66,11 +61,11 @@ class KindRecaller(Protocol): """Source column for cross-encoder rerank passages (display text).""" async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: ... async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: ... @@ -112,90 +107,3 @@ def cosine_score_from_distance(distance: float | None) -> float: if sim > 1.0: return 1.0 return sim - - -def tokenize_query(tokenizer: Tokenizer, query: str) -> str: - """Run the configured tokenizer over the query and join with spaces. - - Cascade joins tokens with a single space when writing the - ``*_tokens`` columns; LanceDB FTS expects a whitespace-tokenised - query string. Same function on both sides keeps BM25 scoring - symmetric. - - Note: prefer :func:`build_or_query` for new code — it sidesteps - the tantivy implicit-AND query-parser pitfall where a single - IDF≈0 token (e.g. an owner's own name on the owner's partition) - poisons the whole query into zero hits. - """ - tokens = tokenizer.tokenize(query) - return " ".join(t for t in tokens if t) - - -def build_or_query( - tokenizer: Tokenizer, - query: str, - column: str, -) -> FullTextQuery | None: - """Tokenise ``query`` and wrap in an OR-mode FTS query. - - Mirrors the enterprise ES design - (``bool.should + minimum_should_match=1``): - - - jieba tokenises the user query. - - Each token becomes its own :class:`MatchQuery` clause. - - Clauses combine under :class:`BooleanQuery` with - :attr:`Occur.SHOULD` so any single matching token surfaces - the document. - - LanceDB still computes a proper joint BM25 score from all - clauses; tokens with IDF ≈ 0 contribute ~0 but no longer - poison the rest of the query. - - Returns ``None`` when the query tokenises to nothing (the - caller must guard against this — there's no useful BM25 query - over an empty token set). - - Single-token queries collapse to a bare :class:`MatchQuery` - (skipping the boolean wrapper) so the FTS engine doesn't pay - for an unnecessary boolean layer. - """ - tokens = [t for t in tokenizer.tokenize(query) if t] - if not tokens: - return None - if len(tokens) == 1: - return MatchQuery(tokens[0], column=column) - clauses: list[tuple[Occur, FullTextQuery]] = [ - (Occur.SHOULD, MatchQuery(t, column=column)) for t in tokens - ] - return BooleanQuery(clauses) - - -def build_or_query_multi_column( - tokenizer: Tokenizer, - query: str, - columns: Sequence[str], -) -> dict[str, FullTextQuery] | None: - """Same as :func:`build_or_query` but emit one FTS query per column. - - ``MatchQuery`` is bound to a single column, and LanceDB FTS only - searches one column per ``nearest_to_text`` call. Dual-column - kinds (``agent_case`` over ``task_intent_tokens`` / - ``approach_tokens``, ``agent_skill`` over - ``description_tokens`` / ``content_tokens``, etc.) need one - OR-bundle per column and merge the results in the caller. - - Returns ``None`` on empty tokenisation; otherwise a dict - ``{column: FullTextQuery}`` ready to feed into separate - ``nearest_to_text`` calls. - """ - tokens = [t for t in tokenizer.tokenize(query) if t] - if not tokens: - return None - out: dict[str, FullTextQuery] = {} - for col in columns: - if len(tokens) == 1: - out[col] = MatchQuery(tokens[0], column=col) - else: - out[col] = BooleanQuery( - [(Occur.SHOULD, MatchQuery(t, column=col)) for t in tokens] - ) - return out diff --git a/src/everos/memory/search/recall/episode.py b/src/everos/memory/search/recall/episode.py index a2766dcec..47db9c1fb 100644 --- a/src/everos/memory/search/recall/episode.py +++ b/src/everos/memory/search/recall/episode.py @@ -7,11 +7,16 @@ from everalgo.types import Candidate -from everos.infra.persistence.lancedb import Episode, get_table +from everos.infra.persistence.index import ( + Episode, + Predicate, + all_of, + episode_repo, + one_of, +) from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -30,10 +35,6 @@ def _inject_parent_id(candidates: list[Candidate]) -> list[Candidate]: ] -def _q(value: str) -> str: - return value.replace("'", "''") - - class EpisodeRecaller: """BM25 + vector recall over the LanceDB ``episode`` table.""" @@ -45,7 +46,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery. @@ -55,12 +56,11 @@ async def sparse_recall( Mirrors enterprise's ``bool.should + minimum_should_match=1`` ES design. """ - bq = build_or_query(self._deps.tokenizer, query, column=Episode.BM25_FIELDS[0]) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await episode_repo.sparse_search( + terms, where, columns=Episode.BM25_FIELDS, limit=limit ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -68,20 +68,11 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query() - .nearest_to(list(vector)) - .column("vector") - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await episode_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -92,19 +83,19 @@ async def dense_recall( ] async def sparse_recall_as_child( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Sparse recall returning episodes as MaxSim child candidates.""" return _inject_parent_id(await self.sparse_recall(query, where, limit=limit)) async def dense_recall_as_child( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Dense recall (body vector ANN) returning as MaxSim children.""" return _inject_parent_id(await self.dense_recall(vector, where, limit=limit)) async def dense_recall_subject( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """ANN over the ``subject_vector`` column. @@ -113,15 +104,8 @@ async def dense_recall_subject( """ if not vector: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query() - .nearest_to(list(vector)) - .column("subject_vector") - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() + rows = await episode_repo.dense_search( + vector, where, limit=limit, vector_field="subject_vector" ) return [ row_to_candidate( @@ -133,13 +117,13 @@ async def dense_recall_subject( ] async def dense_recall_subject_as_child( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Subject-vector ANN returning as MaxSim children.""" candidates = await self.dense_recall_subject(vector, where, limit=limit) return _inject_parent_id(candidates) - async def fetch_all_for_owner(self, where: str) -> list[Candidate]: + async def fetch_all_for_owner(self, where: Predicate) -> list[Candidate]: """Flat scan — all episodes for this owner, keyed by entry_id. Cluster membership matching in ``acluster_retrieve`` compares @@ -149,8 +133,10 @@ async def fetch_all_for_owner(self, where: str) -> list[Candidate]: No ``limit`` — the full owner partition is required for cluster membership matching. """ - table = await get_table(Episode.TABLE_NAME, Episode) - rows = await table.query().where(where).to_list() + rows = [ + record.model_dump(mode="python") + for record in await episode_repo.scan(where) + ] result: list[Candidate] = [] for r in rows: entry_id = r.get("entry_id") @@ -168,13 +154,11 @@ async def fetch_all_for_owner(self, where: str) -> list[Candidate]: return result async def fetch_by_entry_ids( - self, entry_ids: list[str], where: str + self, entry_ids: list[str], where: Predicate ) -> list[Candidate]: """Fetch episodes by entry_id (for facts whose parent_id is an entry_id).""" if not entry_ids: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - quoted = ", ".join(f"'{_q(eid)}'" for eid in entry_ids) - full_where = f"({where}) AND (entry_id IN ({quoted}))" - rows = await table.query().where(full_where).limit(len(entry_ids)).to_list() + full_where = all_of(where, one_of("entry_id", entry_ids)) + rows = await episode_repo.search(where=full_where, limit=len(entry_ids)) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/knowledge_topic.py b/src/everos/memory/search/recall/knowledge_topic.py index ea484e17a..f3ad1229a 100644 --- a/src/everos/memory/search/recall/knowledge_topic.py +++ b/src/everos/memory/search/recall/knowledge_topic.py @@ -12,47 +12,24 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import KnowledgeTopic, get_table +from everos.infra.persistence.index import ( + KnowledgeTopic, + Predicate, + knowledge_topic_repo, +) from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) -def _merge_bm25_results( - per_column: tuple[list[dict], ...], - *, - limit: int, -) -> list[dict]: - """Merge multi-column BM25 results by id, keeping max score.""" - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - return sorted( - best.values(), - key=lambda r: float(r.get("_score", 0.0)), - reverse=True, - )[:limit] - - class KnowledgeTopicRecaller: """BM25 (dual-column) + vector recall over the LanceDB ``knowledge_topic`` table. @@ -68,7 +45,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Dual-column BM25 recall via OR-mode BooleanQuery per column. @@ -78,46 +55,27 @@ async def sparse_recall( matching the query in either its summary or its content body is surfaced without double-counting. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, KnowledgeTopic.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in KnowledgeTopic.BM25_FIELDS), + merged_rows = await knowledge_topic_repo.sparse_search( + terms, + where, + columns=KnowledgeTopic.BM25_FIELDS, + limit=limit, ) - merged_rows = _merge_bm25_results(per_column, limit=limit) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Cosine ANN over the ``summary`` vector (1024-d).""" if not vector: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await knowledge_topic_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/profile.py b/src/everos/memory/search/recall/profile.py index be149b927..640d27091 100644 --- a/src/everos/memory/search/recall/profile.py +++ b/src/everos/memory/search/recall/profile.py @@ -19,7 +19,7 @@ from typing import Any from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import user_profile_repo +from everos.infra.persistence.index import user_profile_repo from ..dto import SearchProfileItem diff --git a/src/everos/memory/strategies/extract_agent_skill.py b/src/everos/memory/strategies/extract_agent_skill.py index 7d4e6bc3b..cb840ea3e 100644 --- a/src/everos/memory/strategies/extract_agent_skill.py +++ b/src/everos/memory/strategies/extract_agent_skill.py @@ -83,10 +83,10 @@ from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase as LanceAgentCase, ) -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, ) diff --git a/src/everos/memory/strategies/extract_user_profile.py b/src/everos/memory/strategies/extract_user_profile.py index 711eca096..38e7db5d4 100644 --- a/src/everos/memory/strategies/extract_user_profile.py +++ b/src/everos/memory/strategies/extract_user_profile.py @@ -59,7 +59,7 @@ from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.events import BaseEvent from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import episode_repo +from everos.infra.persistence.index import episode_repo from everos.infra.persistence.markdown import ( ProfileReader, ProfileWriter, diff --git a/src/everos/memory/strategies/reflect_episodes.py b/src/everos/memory/strategies/reflect_episodes.py index 2d39b8795..44aa0e149 100644 --- a/src/everos/memory/strategies/reflect_episodes.py +++ b/src/everos/memory/strategies/reflect_episodes.py @@ -22,7 +22,7 @@ from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.events import CronTick from everos.infra.ome.triggers import Cron -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( atomic_fact_repo, episode_repo, ) diff --git a/src/everos/service/get.py b/src/everos/service/get.py index 6d5a72bc2..4f246ddee 100644 --- a/src/everos/service/get.py +++ b/src/everos/service/get.py @@ -11,7 +11,7 @@ from __future__ import annotations from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, episode_repo, diff --git a/src/everos/service/knowledge.py b/src/everos/service/knowledge.py index 2040ace17..e36941708 100644 --- a/src/everos/service/knowledge.py +++ b/src/everos/service/knowledge.py @@ -44,6 +44,7 @@ from everos.core.observability.logging import get_logger from everos.core.persistence import MemoryRoot from everos.core.persistence.markdown import dump_frontmatter, parse_frontmatter +from everos.infra.persistence.index import Predicate, all_of, eq from everos.infra.persistence.markdown import ( KnowledgeWriter, ensure_taxonomy, @@ -1148,26 +1149,22 @@ def _validate_scope_id(value: str, name: str) -> None: raise ValueError(f"{name} contains invalid characters: {value!r}") -def compile_knowledge_where(app_id: str, project_id: str) -> str: - """Build a LanceDB ``where`` clause scoped to the given tenant. +def compile_knowledge_where(app_id: str, project_id: str) -> Predicate: + """Build a backend-neutral predicate scoped to the given tenant. Args: app_id: Tenant application identifier. project_id: Tenant project identifier. Returns: - SQL-style predicate string safe for use in LanceDB ``where`` parameter. + Predicate safe for the active derived-index backend. Raises: ValueError: If either id contains invalid characters. """ _validate_scope_id(app_id, "app_id") _validate_scope_id(project_id, "project_id") - - def _esc(v: str) -> str: - return v.replace("'", "''") - - return f"app_id = '{_esc(app_id)}' AND project_id = '{_esc(project_id)}'" + return all_of(eq("app_id", app_id), eq("project_id", project_id)) # ── Recall helpers ─────────────────────────────────────────────────────────── @@ -1175,7 +1172,7 @@ def _esc(v: str) -> str: async def _base_retrieve( recaller: KnowledgeTopicRecaller, - where: str, + where: Predicate, *, method: str, query: str, @@ -1290,7 +1287,7 @@ def _require_search_providers() -> tuple[EmbeddingProvider, RerankProvider]: async def _run_category_pipeline( query: str, - where: str, + where: Predicate, *, method: str, vector: list[float], diff --git a/tests/_consistency_assertions.py b/tests/_consistency_assertions.py index 5890cbaa1..41f0879a4 100644 --- a/tests/_consistency_assertions.py +++ b/tests/_consistency_assertions.py @@ -201,7 +201,7 @@ async def _check_daily_log( ) for entry in parsed.entries } - lance_rows = await spec.lance_repo.find_where( + lance_rows = await spec.index_repo.find_where( f"md_path = '{_q(md_path)}'", limit=10_000 ) lance_sha_by_id = {r.entry_id: r.content_sha256 for r in lance_rows} @@ -246,7 +246,7 @@ async def _check_user_profile( ), } ) - lance_row = await spec.lance_repo.get_by_id(owner_id) + lance_row = await spec.index_repo.get_by_id(owner_id) if lance_row is None: raise AssertionError( f"user_profile row missing for owner {owner_id!r} @ {md_path}" @@ -306,7 +306,7 @@ async def _check_agent_skill( "references_content": references_content, } ) - lance_row = await spec.lance_repo.get_by_id(skill_id) + lance_row = await spec.index_repo.get_by_id(skill_id) if lance_row is None: raise AssertionError( f"agent_skill row missing for skill {skill_id!r} @ {md_path}" diff --git a/tests/integration/test_cascade_all_kinds_consistency.py b/tests/integration/test_cascade_all_kinds_consistency.py index c2310ecf3..74ca489ef 100644 --- a/tests/integration/test_cascade_all_kinds_consistency.py +++ b/tests/integration/test_cascade_all_kinds_consistency.py @@ -6,7 +6,7 @@ * ``frontmatter.entry_count == N`` * number of ```` blocks == N -* ``lance_repo.count_rows(md_path=...) == N`` +* ``index_repo.count_rows(md_path=...) == N`` * lance ``entry_id`` set == md ``entry_id`` set This is the strict counterpart to the loose ``>=`` assertions in diff --git a/tests/integration/test_cascade_cli_integration.py b/tests/integration/test_cascade_cli_integration.py index 22511ddf1..c89ad7f84 100644 --- a/tests/integration/test_cascade_cli_integration.py +++ b/tests/integration/test_cascade_cli_integration.py @@ -124,7 +124,7 @@ def test_sync_with_path_outside_root_errors( # between the two tokens. output = result.stdout + (result.stderr or "") plain_output = _strip_ansi(output) - assert re.search(r"not under[^\w]+memory root", plain_output), output + assert re.search(r"not[^\w]+under[^\w]+memory[^\w]+root", plain_output), output def test_sync_with_unmatched_path( diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py new file mode 100644 index 000000000..f84763270 --- /dev/null +++ b/tests/integration/test_milvus_remote.py @@ -0,0 +1,359 @@ +"""Cross-backend behavior against Milvus Server or Zilliz Cloud. + +Set ``EVEROS_TEST_MILVUS_URI`` and, when required, +``EVEROS_TEST_MILVUS_TOKEN``. The same test is used for self-hosted and cloud +endpoints and creates uniquely prefixed, disposable collections. +""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import os +import uuid + +import pytest +import pytest_asyncio + +from everos.config import load_settings + +_URI = os.environ.get("EVEROS_TEST_MILVUS_URI", "") + +pytestmark = pytest.mark.skipif( + not _URI, + reason="EVEROS_TEST_MILVUS_URI is not configured", +) + + +@pytest_asyncio.fixture(autouse=True) +async def _remote_milvus(monkeypatch: pytest.MonkeyPatch): + prefix = f"everos_e2e_{uuid.uuid4().hex}" + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", _URI) + monkeypatch.setenv( + "EVEROS_MILVUS__TOKEN", + os.environ.get("EVEROS_TEST_MILVUS_TOKEN", ""), + ) + monkeypatch.setenv( + "EVEROS_MILVUS__DB_NAME", + os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", ""), + ) + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", prefix) + load_settings.cache_clear() + + from everos.infra.persistence.index import episode_repo, startup + + try: + if os.environ.get("EVEROS_TEST_MILVUS_FULL_STARTUP") == "1": + await startup() + else: + await episode_repo._repo().ensure_collection() # type: ignore[attr-defined] + yield + finally: + from everos.infra.persistence.index import drop_business_tables, shutdown + + await drop_business_tables() + await shutdown() + load_settings.cache_clear() + + +def _episode( + *, + row_id: str, + entry_id: str, + session_id: str, + text: str, + vector_axis: int, + subject_axis: int, + timestamp: dt.datetime, +): # type: ignore[no-untyped-def] + from everos.infra.persistence.index import Episode + + vector = [0.0] * 1024 + vector[vector_axis] = 1.0 + subject_vector = [0.0] * 1024 + subject_vector[subject_axis] = 1.0 + return Episode( + id=row_id, + entry_id=entry_id, + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + session_id=session_id, + timestamp=timestamp, + parent_id=f"mc_{entry_id}", + sender_ids=["user"], + subject=f"subject {text}", + episode=text, + episode_tokens=text, + md_path="test_app/test_project/users/u1/episodes/day.md", + content_sha256=entry_id, + vector=vector, + subject_vector=subject_vector, + ) + + +async def test_remote_milvus_matches_derived_index_contract() -> None: + from everos.infra.persistence.index import ( + Episode, + UserProfile, + episode_repo, + eq, + is_null, + user_profile_repo, + ) + from everos.memory.search import FilterNode + from everos.memory.search.filters import compile_filters + + first_vector = [1.0] + [0.0] * 1023 + first_subject = [0.0, 1.0] + [0.0] * 1022 + now = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + await episode_repo.upsert( + [ + _episode( + row_id="u1_ep1", + entry_id="ep1", + session_id="abc=", + text="red apple memory", + vector_axis=0, + subject_axis=1, + timestamp=now, + ), + _episode( + row_id="u1_ep2", + entry_id="ep2", + session_id="other", + text="blue banana memory", + vector_axis=1, + subject_axis=0, + timestamp=now + dt.timedelta(seconds=1), + ), + ] + ) + + where = compile_filters( + None, + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + rows = await episode_repo.find_where(where, limit=10) + assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} + assert await episode_repo.count() == 2 + + equals_filter = compile_filters( + FilterNode.model_validate({"session_id": "abc="}), + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + equals_rows = await episode_repo.find_where(equals_filter, limit=10) + assert [row.id for row in equals_rows] == ["u1_ep1"] + + sparse = await episode_repo.sparse_search( + ["apple"], where, columns=Episode.BM25_FIELDS, limit=5 + ) + assert sparse[0]["id"] == "u1_ep1" + assert sparse[0]["_score"] > 0 + + dense = await episode_repo.dense_search(first_vector, where, limit=5) + assert dense[0]["id"] == "u1_ep1" + assert dense[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + by_subject = await episode_repo.dense_search( + first_subject, + where, + limit=5, + vector_field="subject_vector", + ) + assert by_subject[0]["id"] == "u1_ep1" + assert by_subject[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + page, total = await episode_repo.find_where_paginated( + where, + sort_by="timestamp", + page=1, + page_size=1, + ) + assert total == 2 + assert len(page) == 1 + + concurrent = await asyncio.gather( + *(episode_repo.find_where(where, limit=10) for _ in range(4)) + ) + assert all(len(result) == 2 for result in concurrent) + + if os.environ.get("EVEROS_TEST_MILVUS_FULL_STARTUP") == "1": + profile = UserProfile( + id="u1", + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + summary="initial profile", + explicit_info_json="[]", + implicit_traits_json="[]", + profile_timestamp_ms=1, + md_path="test_app/test_project/users/u1/user.md", + content_sha256="profile-v1", + ) + await user_profile_repo.upsert([profile]) + await user_profile_repo.update( + {"summary": "updated profile"}, where=eq("id", "u1") + ) + updated_profile = await user_profile_repo.get_by_id("u1") + assert updated_profile is not None + assert updated_profile.summary == "updated profile" + + # Exercise the logical-null mapping for physical Milvus vector fields and + # the iterator-backed scan path. This also guards against reintroducing the + # old implicit 100-row maintenance cap. + null_vector_rows = [] + for index in range(101): + row = _episode( + row_id=f"u1_null_{index}", + entry_id=f"null_{index}", + session_id="null-vectors", + text=f"unembedded memory {index}", + vector_axis=0, + subject_axis=0, + timestamp=now + dt.timedelta(minutes=index + 1), + ) + null_vector_rows.append( + row.model_copy(update={"vector": None, "subject_vector": None}) + ) + await episode_repo.upsert(null_vector_rows) + assert await episode_repo.count_where(is_null("vector")) == 101 + assert len(await episode_repo.scan()) == 103 + + assert ( + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) + == 103 + ) + assert await episode_repo.count() == 0 + + +async def test_update_preserves_vectors_on_a_row_that_has_them() -> None: + """Milvus has no partial-column update, so update() re-upserts whole rows. + + The only prior coverage ran against ``user_profile``, which carries no + vector column — so the read-back-and-re-upsert of a 1024-d vector (the + path backfill and reflection both take) was never exercised remotely. + """ + from everos.infra.persistence.index import episode_repo, eq + + now = dt.datetime(2026, 3, 1, 12, 0, tzinfo=dt.UTC) + row = _episode( + row_id="u1_vec_update", + entry_id="vec_update", + session_id="vector-update", + text="vector bearing episode", + vector_axis=7, + subject_axis=11, + timestamp=now, + ) + await episode_repo.upsert([row]) + + await episode_repo.update( + {"deprecated_by": "u1_replacement"}, where=eq("id", "u1_vec_update") + ) + + stored = await episode_repo.get_by_id("u1_vec_update") + assert stored is not None + assert stored.deprecated_by == "u1_replacement" + # The untouched columns must survive the round-trip intact. + assert stored.vector is not None + assert stored.subject_vector is not None + assert stored.vector[7] == pytest.approx(1.0) + assert stored.subject_vector[11] == pytest.approx(1.0) + assert stored.timestamp == now + assert stored.sender_ids == ["user"] + assert stored.episode == "vector bearing episode" + + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) + + +async def test_datetime_round_trip_survives_pre_2001_instants() -> None: + """Epoch-ms values below 1e12 must not be re-read as epoch seconds. + + ``to_timestamp_ms`` always writes milliseconds, so the read side has to + parse milliseconds unconditionally. A seconds-vs-ms heuristic sends any + pre-2001-09-09 instant into the year 30000 and raises on the way back. + """ + from everos.infra.persistence.index import episode_repo, eq + + old = dt.datetime(1999, 1, 1, tzinfo=dt.UTC) + row = _episode( + row_id="u1_pre2001", + entry_id="pre2001", + session_id="old-instants", + text="an episode from before the ms/seconds threshold", + vector_axis=3, + subject_axis=5, + timestamp=old, + ) + await episode_repo.upsert([row]) + + stored = await episode_repo.get_by_id("u1_pre2001") + assert stored is not None + assert stored.timestamp == old + + assert await episode_repo.count_where(eq("timestamp", old)) == 1 + + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) + + +async def test_verify_accepts_a_collection_this_adapter_just_created() -> None: + """The decisive check for physical schema verification. + + Every offline test builds its fake ``describe_collection`` reply from the + same descriptor the verifier compares against, so they can only prove that + creation and verification agree with each other. Whether that descriptor + matches what Milvus actually stores and reports back — VARCHAR lengths it + may normalize, flags it may omit, the analyzer it attaches for BM25 — can + only be established against a real server. + + The fixture creates the collections with a fresh prefix, so verification + never runs during setup; dropping the process-local readiness cache forces + the existing-collection path. + """ + from everos.infra.persistence.index import ALL_REPOS, episode_repo + from everos.infra.persistence.milvus.repository import MilvusRepoBase + + await episode_repo.upsert( + [ + _episode( + row_id="u1_verify", + entry_id="verify", + session_id="verify", + text="a row so the collection is not empty", + vector_axis=1, + subject_axis=2, + timestamp=dt.datetime(2026, 5, 1, tzinfo=dt.UTC), + ) + ] + ) + + for repo in ALL_REPOS: + milvus_repo = repo._repo() # type: ignore[attr-defined] + await milvus_repo.ensure_collection() + + MilvusRepoBase._reset_collection_cache() + + for repo in ALL_REPOS: + # Raises MilvusSchemaMismatchError if our declaration and the server's + # description disagree on any field. + await repo._repo().verify_collection() # type: ignore[attr-defined] + + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py index 63b91c2fd..401267ba7 100644 --- a/tests/integration/test_reflection_integration.py +++ b/tests/integration/test_reflection_integration.py @@ -34,6 +34,7 @@ ) from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase from everos.infra.ome.testing import FakeStrategyContext +from everos.infra.persistence.index.lancedb import LanceIndexRepository from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode from everos.infra.persistence.markdown.writers.episode_writer import EpisodeWriter @@ -357,8 +358,8 @@ async def test_reflection_init_merges_cluster_episodes( # -- Build the orchestrator with real repos. orchestrator = ReflectionOrchestrator( cluster_repo=cluster_repo, - episode_store=ep_repo, - atomic_fact_store=af_repo, + episode_store=LanceIndexRepository(ep_repo, LanceEpisode), + atomic_fact_store=LanceIndexRepository(af_repo, AtomicFact), episode_writer=episode_writer, report_repo=reflection_report_repo, reflector=reflector, @@ -563,8 +564,8 @@ async def test_reflection_update_merges_new_episodes_with_existing_merged( orchestrator = ReflectionOrchestrator( cluster_repo=cluster_repo, - episode_store=ep_repo, - atomic_fact_store=af_repo, + episode_store=LanceIndexRepository(ep_repo, LanceEpisode), + atomic_fact_store=LanceIndexRepository(af_repo, AtomicFact), episode_writer=episode_writer, report_repo=reflection_report_repo, reflector=reflector, @@ -609,8 +610,8 @@ async def test_reflection_update_merges_new_episodes_with_existing_merged( # Fresh orchestrator, same FakeLLM (next pop = update_response). orchestrator2 = ReflectionOrchestrator( cluster_repo=cluster_repo, - episode_store=ep_repo, - atomic_fact_store=af_repo, + episode_store=LanceIndexRepository(ep_repo, LanceEpisode), + atomic_fact_store=LanceIndexRepository(af_repo, AtomicFact), episode_writer=episode_writer, report_repo=reflection_report_repo, reflector=reflector, @@ -798,8 +799,8 @@ async def test_reflected_episodes_visible_in_search_deprecated_excluded( orchestrator = ReflectionOrchestrator( cluster_repo=cluster_repo, - episode_store=ep_repo, - atomic_fact_store=af_repo, + episode_store=LanceIndexRepository(ep_repo, LanceEpisode), + atomic_fact_store=LanceIndexRepository(af_repo, AtomicFact), episode_writer=episode_writer, report_repo=reflection_report_repo, reflector=reflector, diff --git a/tests/integration/test_tiers/conftest.py b/tests/integration/test_tiers/conftest.py index b18e9d3f4..27197dfe8 100644 --- a/tests/integration/test_tiers/conftest.py +++ b/tests/integration/test_tiers/conftest.py @@ -40,6 +40,8 @@ import asyncio import importlib import json +import os +import uuid from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager from pathlib import Path @@ -175,6 +177,54 @@ def _reset_search_singletons(monkeypatch: pytest.MonkeyPatch) -> None: # --------------------------------------------------------------------------- +def _index_backends() -> list[str]: + """Backends the tier suites run against. + + LanceDB is always exercised. Milvus joins only when a real server is + configured — it is a remote service, so there is no embedded fallback to + silently degrade to. A run without ``EVEROS_TEST_MILVUS_URI`` therefore + proves nothing about Milvus, and says so by not collecting those cases. + """ + backends = ["lancedb"] + if os.environ.get("EVEROS_TEST_MILVUS_URI"): + backends.append("milvus") + return backends + + +@pytest_asyncio.fixture(params=_index_backends(), ids=lambda b: f"index={b}") +async def index_backend( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> AsyncIterator[str]: + """Point the derived index at one backend for the duration of a test.""" + backend = request.param + monkeypatch.setenv("EVEROS_INDEX__BACKEND", backend) + if backend == "milvus": + monkeypatch.setenv("EVEROS_MILVUS__URI", os.environ["EVEROS_TEST_MILVUS_URI"]) + monkeypatch.setenv( + "EVEROS_MILVUS__TOKEN", os.environ.get("EVEROS_TEST_MILVUS_TOKEN", "") + ) + monkeypatch.setenv( + "EVEROS_MILVUS__DB_NAME", os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", "") + ) + # A fresh prefix per test: collections are remote and outlive the + # process, so two tests sharing one would leak state into each other. + monkeypatch.setenv( + "EVEROS_MILVUS__COLLECTION_PREFIX", f"everos_tier_{uuid.uuid4().hex}" + ) + + yield backend + + if backend == "milvus": + from everos.config import load_settings + from everos.infra.persistence.index import drop_business_tables, shutdown + + load_settings.cache_clear() + try: + await drop_business_tables() + finally: + await shutdown() + + @asynccontextmanager async def _tier_client( tmp_path: Path, @@ -234,7 +284,7 @@ async def _tier_client( @pytest_asyncio.fixture async def tier1_runtime( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, index_backend: str ) -> AsyncIterator[httpx.AsyncClient]: """Tier 1: LLM only. embed/rerank/multimodal all unavailable.""" async with _tier_client( @@ -245,7 +295,7 @@ async def tier1_runtime( @pytest_asyncio.fixture async def tier2_runtime( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, index_backend: str ) -> AsyncIterator[httpx.AsyncClient]: """Tier 2: LLM + embed. rerank unavailable.""" async with _tier_client( @@ -256,7 +306,7 @@ async def tier2_runtime( @pytest_asyncio.fixture async def tier3_runtime( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, index_backend: str ) -> AsyncIterator[httpx.AsyncClient]: """Tier 3: LLM + embed + rerank, all available.""" async with _tier_client( @@ -384,7 +434,7 @@ async def seed_atomic_fact_for_episode( import hashlib from everos.component.utils.datetime import get_utc_now - from everos.infra.persistence.lancedb import AtomicFact, atomic_fact_repo + from everos.infra.persistence.index import AtomicFact, atomic_fact_repo entry_id = f"af_seed_{episode_row['entry_id']}" owner_id = episode_row["owner_id"] @@ -434,3 +484,16 @@ def add_payload( } ], } + + +async def episode_rows(owner_id: str) -> list[dict[str, Any]]: + """Every episode row for one owner, as plain dicts. + + Backend-neutral on purpose: the tier suites assert on indexed content, not + on how a particular engine stores it, so they read through the same port + the application uses. + """ + from everos.infra.persistence.index import episode_repo, eq + + rows = await episode_repo.scan(eq("owner_id", owner_id)) + return [row.model_dump(mode="python") for row in rows] diff --git a/tests/integration/test_tiers/test_tier1_keyword_only.py b/tests/integration/test_tiers/test_tier1_keyword_only.py index 9dacbb506..2cdc0ce95 100644 --- a/tests/integration/test_tiers/test_tier1_keyword_only.py +++ b/tests/integration/test_tiers/test_tier1_keyword_only.py @@ -12,15 +12,7 @@ import pytest from httpx import AsyncClient -from everos.infra.persistence.lancedb import Episode, get_table - -from .conftest import add_and_flush - - -async def _episode_rows(owner_id: str) -> list[dict]: - table = await get_table(Episode.TABLE_NAME, Episode) - return await table.query().where(f"owner_id = '{owner_id}'").to_list() - +from .conftest import add_and_flush, episode_rows # --------------------------------------------------------------------------- # 1. POST /memory/add -> md write + LanceDB vector=NULL @@ -31,7 +23,7 @@ async def test_add_memory_writes_null_vector(tier1_runtime: AsyncClient) -> None body = await add_and_flush(tier1_runtime, session_id="s_tier1_add") assert body["data"]["status"] == "extracted" - rows = await _episode_rows("u_alice") + rows = await episode_rows("u_alice") assert rows, "expected cascade to index the new episode into LanceDB" assert rows[0]["vector"] is None, "Tier 1 (no embed) must write vector=NULL" diff --git a/tests/integration/test_tiers/test_tier2_no_rerank.py b/tests/integration/test_tiers/test_tier2_no_rerank.py index bb7574edd..ae20cc5ba 100644 --- a/tests/integration/test_tiers/test_tier2_no_rerank.py +++ b/tests/integration/test_tiers/test_tier2_no_rerank.py @@ -15,18 +15,11 @@ import pytest from httpx import AsyncClient -from everos.infra.persistence.lancedb import Episode, get_table - -from .conftest import add_and_flush, seed_atomic_fact_for_episode +from .conftest import add_and_flush, episode_rows, seed_atomic_fact_for_episode _STUB_VECTOR = [0.1] * 1024 -async def _episode_rows(owner_id: str) -> list[dict]: - table = await get_table(Episode.TABLE_NAME, Episode) - return await table.query().where(f"owner_id = '{owner_id}'").to_list() - - # --------------------------------------------------------------------------- # 1. Memory add + KEYWORD / VECTOR / HYBRID (user) all succeed # --------------------------------------------------------------------------- @@ -35,7 +28,7 @@ async def _episode_rows(owner_id: str) -> list[dict]: async def test_add_memory_writes_real_vector(tier2_runtime: AsyncClient) -> None: await add_and_flush(tier2_runtime, session_id="s_tier2_add") - rows = await _episode_rows("u_alice") + rows = await episode_rows("u_alice") assert rows, "expected cascade to index the new episode into LanceDB" assert rows[0]["vector"] is not None, "Tier 2 (embed available) must embed" assert len(rows[0]["vector"]) == 1024 @@ -48,7 +41,7 @@ async def test_user_search_methods_succeed( await add_and_flush(tier2_runtime, session_id=f"s_tier2_{method}") # VECTOR recalls via atomic_fact MaxSim, not a direct episode ANN scan # -- see seed_atomic_fact_for_episode's docstring. - episode_row = (await _episode_rows("u_alice"))[0] + episode_row = (await episode_rows("u_alice"))[0] await seed_atomic_fact_for_episode(episode_row, vector=_STUB_VECTOR) resp = await tier2_runtime.post( diff --git a/tests/integration/test_tiers/test_tier3_full.py b/tests/integration/test_tiers/test_tier3_full.py index 41dc0ecd8..01850e580 100644 --- a/tests/integration/test_tiers/test_tier3_full.py +++ b/tests/integration/test_tiers/test_tier3_full.py @@ -16,18 +16,16 @@ from everalgo.types import KnowledgeMemory from httpx import AsyncClient -from everos.infra.persistence.lancedb import Episode, get_table - -from .conftest import add_and_flush, cascade_progress, seed_atomic_fact_for_episode +from .conftest import ( + add_and_flush, + cascade_progress, + episode_rows, + seed_atomic_fact_for_episode, +) _STUB_VECTOR = [0.1] * 1024 -async def _episode_rows(owner_id: str) -> list[dict]: - table = await get_table(Episode.TABLE_NAME, Episode) - return await table.query().where(f"owner_id = '{owner_id}'").to_list() - - # --------------------------------------------------------------------------- # 1. Memory + every search method -> 200 # --------------------------------------------------------------------------- @@ -36,7 +34,7 @@ async def _episode_rows(owner_id: str) -> list[dict]: async def test_add_memory_writes_real_vector(tier3_runtime: AsyncClient) -> None: await add_and_flush(tier3_runtime, session_id="s_tier3_add") - rows = await _episode_rows("u_alice") + rows = await episode_rows("u_alice") assert rows and rows[0]["vector"] is not None @@ -45,7 +43,7 @@ async def test_user_search_methods_succeed( tier3_runtime: AsyncClient, method: str ) -> None: await add_and_flush(tier3_runtime, session_id=f"s_tier3_{method}") - episode_row = (await _episode_rows("u_alice"))[0] + episode_row = (await episode_rows("u_alice"))[0] await seed_atomic_fact_for_episode(episode_row, vector=_STUB_VECTOR) resp = await tier3_runtime.post( diff --git a/tests/integration/test_tiers/test_upgrade_path.py b/tests/integration/test_tiers/test_upgrade_path.py index 7d8c2425f..72a35bb5a 100644 --- a/tests/integration/test_tiers/test_upgrade_path.py +++ b/tests/integration/test_tiers/test_upgrade_path.py @@ -16,6 +16,7 @@ from __future__ import annotations +import asyncio import hashlib from pathlib import Path @@ -41,6 +42,18 @@ async def _episode_rows(owner_id: str) -> list[dict]: return await table.query().where(f"owner_id = '{owner_id}'").to_list() +async def _wait_for_episode_rows( + owner_id: str, expected: int, *, deadline_seconds: float +) -> list[dict]: + """Wait for every logical entry in a debounced daily-log update.""" + async with asyncio.timeout(deadline_seconds): + while True: + rows = await _episode_rows(owner_id) + if len(rows) >= expected: + return rows + await asyncio.sleep(0.1) + + async def _atomic_fact_rows(owner_id: str) -> list[dict]: table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) return await table.query().where(f"owner_id = '{owner_id}'").to_list() @@ -118,7 +131,10 @@ async def test_tier1_to_tier2_upgrade_via_backfill( # below directly against the LanceDB table. await wait_drained(deadline_seconds=40.0) - rows = await _episode_rows("u_alice") + # The queue drains file-level work, but another append to the same + # daily log can arrive just after that drain. Wait for the logical + # entry count as the end-to-end completion condition. + rows = await _wait_for_episode_rows("u_alice", _N_ITEMS, deadline_seconds=40.0) assert len(rows) == _N_ITEMS assert all(r["vector"] is None for r in rows), ( "Tier 1 (no embed) must write every episode with vector=NULL" diff --git a/tests/unit/test_config/test_settings.py b/tests/unit/test_config/test_settings.py index fb4fbb17f..6d3f2c59d 100644 --- a/tests/unit/test_config/test_settings.py +++ b/tests/unit/test_config/test_settings.py @@ -120,6 +120,30 @@ def test_embedding_rerank_defaults() -> None: assert s.llm.api_key.get_secret_value() == "" +def test_index_milvus_defaults() -> None: + s = Settings() + assert s.index.backend == "lancedb" + assert s.milvus.uri == "" + assert s.milvus.token.get_secret_value() == "" + assert s.milvus.db_name == "" + assert s.milvus.consistency_level == "Session" + assert s.milvus.collection_prefix == "everos" + + +def test_index_milvus_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", "http://localhost:19530") + monkeypatch.setenv("EVEROS_MILVUS__TOKEN", "secret") + monkeypatch.setenv("EVEROS_MILVUS__DB_NAME", "tenant_a") + monkeypatch.setenv("EVEROS_MILVUS__CONSISTENCY_LEVEL", "Strong") + s = Settings() + assert s.index.backend == "milvus" + assert s.milvus.uri == "http://localhost:19530" + assert s.milvus.token.get_secret_value() == "secret" + assert s.milvus.db_name == "tenant_a" + assert s.milvus.consistency_level == "Strong" + + def test_resolve_root_default(monkeypatch: pytest.MonkeyPatch) -> None: """No --root, no EVEROS_ROOT → ~/.everos.""" monkeypatch.delenv("EVEROS_ROOT", raising=False) diff --git a/tests/unit/test_entrypoints/test_api/test_lifespans/test_lancedb_unbackfilled_hint.py b/tests/unit/test_entrypoints/test_api/test_lifespans/test_lancedb_unbackfilled_hint.py index f8a5cc84a..180b16cc4 100644 --- a/tests/unit/test_entrypoints/test_api/test_lifespans/test_lancedb_unbackfilled_hint.py +++ b/tests/unit/test_entrypoints/test_api/test_lifespans/test_lancedb_unbackfilled_hint.py @@ -29,18 +29,20 @@ import structlog.testing from everos.entrypoints.api.lifespans import lancedb as lancedb_lifespan -from everos.infra.persistence.lancedb import BUSINESS_SCHEMAS_WITH_VECTOR +from everos.infra.persistence.index import ALL_REPOS +from everos.infra.persistence.index.schema import schema_for -class _FakeTable: - """Minimal stand-in for :class:`BaseLanceTable` used by the hint.""" +class _FakeRepo: + """Minimal backend-neutral repository used by the hint.""" - def __init__(self, null_count: int) -> None: + def __init__(self, schema: Any, null_count: int) -> None: + self.schema = schema self._null_count = null_count - self.count_rows_calls = 0 + self.count_where_calls = 0 - async def count_rows(self, filter: str) -> int: - self.count_rows_calls += 1 + async def count_where(self, predicate: Any) -> int: + self.count_where_calls += 1 return self._null_count @@ -53,18 +55,15 @@ def _isolated_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: def _wire_tables( monkeypatch: pytest.MonkeyPatch, *, null_count: int -) -> dict[str, _FakeTable]: - """Replace ``get_table`` in the lifespan module with a per-table stub.""" - tables: dict[str, _FakeTable] = { - schema.TABLE_NAME: _FakeTable(null_count) - for schema in BUSINESS_SCHEMAS_WITH_VECTOR +) -> dict[str, _FakeRepo]: + """Replace the facade registry with per-schema repository stubs.""" + repos: dict[str, _FakeRepo] = { + repo.schema.TABLE_NAME: _FakeRepo(repo.schema, null_count) + for repo in ALL_REPOS + if schema_for(repo.schema).vector_fields } - - async def _fake_get_table(name: str, _schema: Any) -> _FakeTable: - return tables[name] - - monkeypatch.setattr(lancedb_lifespan, "get_table", _fake_get_table) - return tables + monkeypatch.setattr(lancedb_lifespan, "ALL_REPOS", tuple(repos.values())) + return repos async def test_hint_fires_when_null_vectors_exist( @@ -77,10 +76,10 @@ async def test_hint_fires_when_null_vectors_exist( emissions = [e for e in captured if e.get("event") == "unbackfilled_memory_rows"] assert len(emissions) == 1 - expected_total = 3 * len(BUSINESS_SCHEMAS_WITH_VECTOR) + expected_total = 3 * len(tables) assert emissions[0]["count"] == expected_total # Every business table contributes exactly one ``count_rows`` call. - assert all(t.count_rows_calls == 1 for t in tables.values()) + assert all(t.count_where_calls == 1 for t in tables.values()) async def test_hint_silent_when_no_null_vectors( @@ -95,24 +94,24 @@ async def test_hint_silent_when_no_null_vectors( assert emissions == [] # Every table was scanned (unconditional count) — none had rows to # report, so no banner. No marker involved. - assert all(t.count_rows_calls == 1 for t in tables.values()) + assert all(t.count_where_calls == 1 for t in tables.values()) async def test_per_table_failure_is_swallowed_and_logged( _isolated_root: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - tables: dict[str, _FakeTable] = { - schema.TABLE_NAME: _FakeTable(null_count=1) - for schema in BUSINESS_SCHEMAS_WITH_VECTOR + tables: dict[str, _FakeRepo] = { + repo.schema.TABLE_NAME: _FakeRepo(repo.schema, null_count=1) + for repo in ALL_REPOS + if schema_for(repo.schema).vector_fields } - poisoned = BUSINESS_SCHEMAS_WITH_VECTOR[0].TABLE_NAME + poisoned = next(iter(tables)) - async def _fake_get_table(name: str, _schema: Any) -> _FakeTable: - if name == poisoned: - raise RuntimeError("simulated LanceDB hiccup") - return tables[name] + async def _poisoned_count(_predicate: Any) -> int: + raise RuntimeError("simulated index hiccup") - monkeypatch.setattr(lancedb_lifespan, "get_table", _fake_get_table) + tables[poisoned].count_where = _poisoned_count # type: ignore[method-assign] + monkeypatch.setattr(lancedb_lifespan, "ALL_REPOS", tuple(tables.values())) with structlog.testing.capture_logs() as captured: await lancedb_lifespan._log_unbackfilled_hint() @@ -125,4 +124,4 @@ async def _fake_get_table(name: str, _schema: Any) -> _FakeTable: # out silently. emissions = [e for e in captured if e.get("event") == "unbackfilled_memory_rows"] assert len(emissions) == 1 - assert emissions[0]["count"] == len(BUSINESS_SCHEMAS_WITH_VECTOR) - 1 + assert emissions[0]["count"] == len(tables) - 1 diff --git a/tests/unit/test_infra/test_index_contract.py b/tests/unit/test_infra/test_index_contract.py new file mode 100644 index 000000000..10b3a274d --- /dev/null +++ b/tests/unit/test_infra/test_index_contract.py @@ -0,0 +1,107 @@ +"""Contract tests for the backend-neutral derived-index boundary.""" + +from __future__ import annotations + +import datetime as dt +from pathlib import Path + +import pytest + +from everos.infra.persistence.index import ( + ALL_REPOS, + Episode, + IndexBackend, + IndexRepository, + all_of, + episode_repo, + eq, +) +from everos.infra.persistence.index.lancedb import lance_index_backend, render_predicate +from everos.infra.persistence.index.schema import IndexFieldKind, schema_for +from everos.infra.persistence.lancedb import lancedb_manager + + +@pytest.fixture(autouse=True) +async def _isolated_index(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + lancedb_manager._conn = None + lancedb_manager._tables.clear() + yield + await lancedb_manager.dispose_connection() + + +def _episode(number: int, *, owner_id: str = "owner") -> Episode: + return Episode( + id=f"{owner_id}_ep_{number:04d}", + entry_id=f"ep_{number:04d}", + owner_id=owner_id, + owner_type="user", + session_id="session", + timestamp=dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + dt.timedelta(seconds=number), + parent_id="memcell", + sender_ids=[owner_id], + subject="subject", + summary="summary", + episode=f"portable record {number}", + episode_tokens=f"portable record {number}", + md_path=f"users/{owner_id}/episodes/episode.md", + content_sha256=f"{number:064x}", + ) + + +def test_all_registered_repositories_and_backend_satisfy_ports() -> None: + assert isinstance(lance_index_backend, IndexBackend) + assert {repo.table_name for repo in lance_index_backend.repositories} == { + repo.table_name for repo in ALL_REPOS + } + assert len({repo.table_name for repo in ALL_REPOS}) == len(ALL_REPOS) == 7 + assert all(isinstance(repo, IndexRepository) for repo in ALL_REPOS) + + +def test_all_logical_schemas_are_portable_and_validate_bm25_fields() -> None: + for repo in ALL_REPOS: + logical = schema_for(repo.schema) + assert logical.table_name == repo.table_name + declared = {field.name for field in logical.fields} + assert set(logical.bm25_fields) <= declared + for field in logical.vector_fields: + assert field.kind is IndexFieldKind.DENSE_VECTOR + assert field.dimension == 1024 + + +def test_lance_predicate_renderer_owns_escaping() -> None: + rendered = render_predicate( + all_of(eq("owner_id", "o'reilly"), eq("session_id", "session")) + ) + assert "owner_id = 'o''reilly'" in rendered + assert "session_id = 'session'" in rendered + + +async def test_repository_port_crud_count_and_uncapped_scan() -> None: + rows = [_episode(number) for number in range(101)] + await episode_repo.upsert(rows) + + owner = eq("owner_id", "owner") + assert await episode_repo.count() == 101 + assert await episode_repo.count_where(owner) == 101 + assert len(await episode_repo.scan(owner)) == 101 + + found = await episode_repo.find_one_where(eq("id", rows[0].id)) + assert found is not None and found.entry_id == rows[0].entry_id + + await episode_repo.update( + {"subject": "updated"}, + where=eq("id", rows[0].id), + ) + updated = await episode_repo.get_by_id(rows[0].id) + assert updated is not None and updated.subject == "updated" + + await episode_repo.delete(eq("id", rows[0].id)) + assert await episode_repo.count_where(owner) == 100 + + +async def test_repository_maintenance_contract_is_executable() -> None: + await episode_repo.upsert([_episode(1)]) + await episode_repo.optimize() + await episode_repo.prune(dt.timedelta(days=7)) + await episode_repo.rebuild_indexes() diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py new file mode 100644 index 000000000..15c5a5e98 --- /dev/null +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -0,0 +1,427 @@ +"""Unit coverage for the remote Milvus derived-index adapter.""" + +from __future__ import annotations + +import datetime as dt + +import pytest +from pymilvus import DataType + +from everos.config import load_settings +from everos.infra.persistence.index import ( + Episode, + IndexRepository, + episode_repo, + foresight_repo, + is_null, + user_profile_repo, +) +from everos.infra.persistence.index.schema import schema_for +from everos.infra.persistence.milvus import repository +from everos.infra.persistence.milvus.milvus_manager import ( + MilvusConfigurationError, + MilvusSchemaMismatchError, + _resolve_uri, +) +from everos.infra.persistence.milvus.predicate import render_predicate +from everos.infra.persistence.milvus.repos import ALL_REPOS +from everos.infra.persistence.milvus.repository import ( + MilvusRepoBase, + MilvusValueLimitError, +) + + +@pytest.fixture(autouse=True) +def _reset_state(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", "http://milvus.example:19530") + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", "unit_test") + load_settings.cache_clear() + MilvusRepoBase._reset_locks_for_tests() + yield + MilvusRepoBase._reset_locks_for_tests() + load_settings.cache_clear() + + +def _describe_response(milvus_repo, *, always_params: bool = False): # type: ignore[no-untyped-def] + """What Milvus reports for the collection this repo would create. + + Built from pymilvus' own ``CollectionSchema.to_dict()`` rather than a + hand-written literal, so the field shape comes from the library instead of + from our assumptions. ``describe_collection`` always emits ``params`` + while ``to_dict`` omits it when empty — ``always_params`` covers that one + known difference between the two serializations. + """ + fields = [ + dict(f) for f in milvus_repo._build_collection_schema().to_dict()["fields"] + ] + if always_params: + for field in fields: + field.setdefault("params", {}) + return {"fields": fields} + + +def _patch_describe(monkeypatch, response): # type: ignore[no-untyped-def] + class _FakeClient: + def describe_collection(self, name: str): # type: ignore[no-untyped-def] + return response + + async def _fake_get_client(): # type: ignore[no-untyped-def] + return _FakeClient() + + monkeypatch.setattr(repository, "get_client", _fake_get_client) + + +def _episode(**overrides): # type: ignore[no-untyped-def] + values = { + "id": "u1_ep1", + "entry_id": "ep1", + "owner_id": "u1", + "owner_type": "user", + "session_id": "abc=", + "timestamp": dt.datetime(2026, 1, 1, tzinfo=dt.UTC), + "parent_id": "mc1", + "sender_ids": ["user"], + "episode": "red apple memory", + "episode_tokens": "red apple memory", + "md_path": "default_app/default_project/users/u1/episodes/day.md", + "content_sha256": "a", + "vector": [1.0] + [0.0] * 1023, + "subject_vector": [0.0, 1.0] + [0.0] * 1022, + } + values.update(overrides) + return Episode(**values) + + +def test_remote_uri_is_required_and_local_paths_are_rejected() -> None: + settings = load_settings().milvus.model_copy(update={"uri": ""}) + with pytest.raises(MilvusConfigurationError, match="requires"): + _resolve_uri(settings) + + settings = settings.model_copy(update={"uri": "/tmp/milvus.db"}) + with pytest.raises(MilvusConfigurationError, match="remote http"): + _resolve_uri(settings) + + settings = settings.model_copy(update={"uri": "file:///tmp/milvus.db"}) + with pytest.raises(MilvusConfigurationError, match=r"http\(s\)"): + _resolve_uri(settings) + + +def test_neutral_schema_tracks_every_model_field_and_dense_vector() -> None: + schema = schema_for(Episode) + assert {field.name for field in schema.fields} == set(Episode.model_fields) + assert [field.name for field in schema.vector_fields] == [ + "vector", + "subject_vector", + ] + assert all(isinstance(repo, IndexRepository) for repo in ALL_REPOS) + + +def test_null_vector_predicate_targets_presence_marker() -> None: + assert ( + render_predicate(is_null("vector"), vector_fields={"vector"}) + == "vector__present == false" + ) + + +def test_record_conversion_stores_every_dense_vector_with_presence() -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + record = milvus_repo._to_milvus_record(_episode()) + assert len(record["vector"]) == 1024 + assert len(record["subject_vector"]) == 1024 + assert record["vector__present"] is True + assert record["subject_vector__present"] is True + + missing = milvus_repo._to_milvus_record(_episode(subject_vector=None)) + assert missing["subject_vector__present"] is False + assert missing["subject_vector"] == [0.0] * 1024 + + +def test_datetime_round_trip_is_exact_below_the_ms_heuristic() -> None: + """Epoch ms are written unconditionally, so they must be read the same way. + + ``from_timestamp`` treats anything under 1e12 as *seconds*, so a + pre-2001-09-09 instant stored as ms comes back in the year 30000 — or + raises ``ValueError: year ... is out of range`` on the way out. The + physical datetime column has to bypass that heuristic entirely. + """ + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + for moment in ( + dt.datetime(1970, 1, 2, tzinfo=dt.UTC), + dt.datetime(1999, 1, 1, tzinfo=dt.UTC), + dt.datetime(2001, 9, 8, tzinfo=dt.UTC), + dt.datetime(2026, 1, 1, tzinfo=dt.UTC), + ): + stored = milvus_repo._to_milvus_record(_episode(timestamp=moment)) + assert "timestamp_ms" in stored + restored = milvus_repo._restore_row(stored) + assert restored["timestamp"] == moment, f"round-trip broke at {moment}" + + +def test_update_fetch_preserves_dummy_vector_for_scalar_only_tables() -> None: + milvus_repo = user_profile_repo._repo() # type: ignore[attr-defined] + assert "_everos_dummy_vector" in milvus_repo._output_fields(include_vectors=True) + assert "_everos_dummy_vector" not in milvus_repo._output_fields( + include_vectors=False + ) + + +def test_record_conversion_reports_varchar_array_and_vector_limits() -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + with pytest.raises(MilvusValueLimitError, match=r"episode is .* UTF-8 bytes"): + milvus_repo._to_milvus_record(_episode(episode="x" * 65_536)) + with pytest.raises(MilvusValueLimitError, match="sender_ids has 257 items"): + milvus_repo._to_milvus_record(_episode(sender_ids=["u"] * 257)) + with pytest.raises(MilvusValueLimitError, match="dimension 2"): + milvus_repo._validate_vector( + milvus_repo.index_schema.field("vector"), + [1.0, 0.0], + ) + + +def test_server_score_normalization() -> None: + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(1.5) == pytest.approx(1.5) + + +async def test_collection_metadata_is_cached_after_startup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + + class _FakeClient: + has_calls = 0 + describe_calls = 0 + + def has_collection(self, name: str) -> bool: + self.has_calls += 1 + return True + + def describe_collection(self, name: str): # type: ignore[no-untyped-def] + self.describe_calls += 1 + return _describe_response(milvus_repo) + + client = _FakeClient() + + async def _fake_get_client(): # type: ignore[no-untyped-def] + return client + + monkeypatch.setattr(repository, "get_client", _fake_get_client) + await milvus_repo.ensure_collection() + await milvus_repo.ensure_collection() + assert client.has_calls == 1 + assert client.describe_calls == 1 + + +# ── Physical schema verification ──────────────────────────────────────── + + +@pytest.mark.parametrize("always_params", [False, True]) +async def test_verify_accepts_every_collection_it_would_create( + monkeypatch: pytest.MonkeyPatch, always_params: bool +) -> None: + """The false-positive guard: our own collections must always pass. + + A checker that rejects a healthy deployment is far worse than the + name-only check it replaces, so every table is verified against the exact + schema this adapter would have created for it. + """ + for repo in ALL_REPOS: + _patch_describe( + monkeypatch, _describe_response(repo, always_params=always_params) + ) + await repo.verify_collection() + + +async def test_verify_rejects_datatype_drift(monkeypatch: pytest.MonkeyPatch) -> None: + """Same field name, wrong physical type — the case a name check waves through.""" + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + for field in response["fields"]: + if field["name"] == "timestamp_ms": + field["type"] = DataType.VARCHAR + _patch_describe(monkeypatch, response) + + with pytest.raises(MilvusSchemaMismatchError, match="timestamp_ms: datatype"): + await milvus_repo.verify_collection() + + +async def test_verify_rejects_vector_dimension_drift( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + for field in response["fields"]: + if field["name"] == "vector": + field["params"] = {"dim": 1} + _patch_describe(monkeypatch, response) + + with pytest.raises( + MilvusSchemaMismatchError, match=r"vector: dim 1 != expected 1024" + ): + await milvus_repo.verify_collection() + + +async def test_verify_rejects_primary_and_nullable_drift( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``describe_collection`` omits both flags when false, so absence is exact.""" + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + for field in response["fields"]: + if field["name"] == "id": + field.pop("is_primary", None) + if field["name"] == "deprecated_by": + field.pop("nullable", None) + _patch_describe(monkeypatch, response) + + with pytest.raises(MilvusSchemaMismatchError) as excinfo: + await milvus_repo.verify_collection() + assert "id: is_primary" in str(excinfo.value) + assert "deprecated_by: nullable" in str(excinfo.value) + + +async def test_verify_rejects_missing_and_stale_columns( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + response["fields"] = [f for f in response["fields"] if f["name"] != "subject"] + response["fields"].append({"name": "left_over", "type": DataType.VARCHAR}) + _patch_describe(monkeypatch, response) + + with pytest.raises(MilvusSchemaMismatchError) as excinfo: + await milvus_repo.verify_collection() + assert "'subject'" in str(excinfo.value) + assert "'left_over'" in str(excinfo.value) + + +async def test_verify_treats_limit_drift_as_advisory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A wrong VARCHAR length fails loudly on write, so it must not block startup. + + Blocking here would turn a server-side normalization we cannot predict into + an outage; the datatype and dimension checks carry the load instead. + """ + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + for field in response["fields"]: + if field["name"] == "subject": + field["params"] = {"max_length": 4096} + _patch_describe(monkeypatch, response) + + warnings: list[tuple[str, dict]] = [] + monkeypatch.setattr( + repository.logger, + "warning", + lambda event, **kw: warnings.append((event, kw)), + ) + + await milvus_repo.verify_collection() + + # Tolerated, but never silently: the drift has to reach the operator. + assert [event for event, _ in warnings] == ["milvus_collection_limit_drift"] + assert warnings[0][1]["details"] == ["subject: max_length 4096 != declared 65535"] + + +# Written out by hand on purpose. _describe_response() builds its fake server +# reply from _build_collection_schema(), which shares _physical_fields() with +# the verifier — so those tests can only prove that creation and verification +# agree with *each other*, never that either matches what we intend. This +# literal is the independent statement of intent: a slip in _physical_for +# (a nullable flipped, a dim dropped, an analyzer lost) changes both sides at +# once and is invisible everywhere except here. +# +# Only a run against a real Milvus can prove the declaration matches what the +# server actually stores; see tests/integration/test_milvus_remote.py. +_EPISODE_PHYSICAL = ( + ("created_at_ms", DataType.INT64), + ("updated_at_ms", DataType.INT64), + ("id", DataType.VARCHAR, "is_primary", 512), + ("entry_id", DataType.VARCHAR, "", 65535), + ("owner_id", DataType.VARCHAR, "", 65535), + ("owner_type", DataType.VARCHAR, "", 65535), + ("app_id", DataType.VARCHAR, "", 65535), + ("project_id", DataType.VARCHAR, "", 65535), + ("session_id", DataType.VARCHAR, "nullable", 65535), + ("timestamp_ms", DataType.INT64), + ("parent_type", DataType.VARCHAR, "", 65535), + ("parent_id", DataType.VARCHAR, "", 65535), + ("sender_ids", DataType.ARRAY, "", 512), + ("subject", DataType.VARCHAR, "nullable", 65535), + ("summary", DataType.VARCHAR, "nullable", 65535), + ("episode", DataType.VARCHAR, "", 65535), + ("episode_tokens", DataType.VARCHAR, "enable_analyzer", 65535), + ("md_path", DataType.VARCHAR, "", 65535), + ("content_sha256", DataType.VARCHAR, "", 65535), + ("deprecated_by", DataType.VARCHAR, "nullable", 65535), + ("vector", DataType.FLOAT_VECTOR, "", None), + ("vector__present", DataType.BOOL), + ("subject_vector", DataType.FLOAT_VECTOR, "", None), + ("subject_vector__present", DataType.BOOL), + ("episode_tokens__sparse", DataType.SPARSE_FLOAT_VECTOR), +) + + +def _flag(field) -> str: # type: ignore[no-untyped-def] + for candidate in ("is_primary", "nullable", "enable_analyzer"): + if getattr(field, candidate): + return candidate + return "" + + +def test_episode_physical_layout_matches_the_declared_snapshot() -> None: + """Pin the physical column list against a hand-written expectation.""" + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + actual = milvus_repo._physical_fields() + + assert [f.name for f in actual] == [row[0] for row in _EPISODE_PHYSICAL] + for field, row in zip(actual, _EPISODE_PHYSICAL, strict=True): + assert field.datatype == row[1], field.name + if len(row) == 2: + assert not _flag(field), field.name + continue + assert _flag(field) == row[2], field.name + assert field.max_length == row[3], field.name + + sender_ids = next(f for f in actual if f.name == "sender_ids") + assert sender_ids.element_type is DataType.VARCHAR + assert sender_ids.max_capacity == 256 + for name in ("vector", "subject_vector"): + assert next(f for f in actual if f.name == name).dim == 1024 + + +def test_vectorless_table_declares_the_dummy_vector_column() -> None: + """Milvus needs at least one vector column, so scalar-only tables fake one.""" + milvus_repo = user_profile_repo._repo() # type: ignore[attr-defined] + actual = {f.name: f for f in milvus_repo._physical_fields()} + + assert "_everos_dummy_vector" in actual + assert actual["_everos_dummy_vector"].datatype is DataType.FLOAT_VECTOR + assert actual["_everos_dummy_vector"].dim == 2 + assert not [ + f for f in actual.values() if f.datatype is DataType.SPARSE_FLOAT_VECTOR + ] + assert actual["id"].is_primary is True + assert actual["summary"].nullable is False + + +def test_nullable_bm25_input_is_declared_not_null() -> None: + """``foresight.evidence_tokens`` is the only nullable BM25 column. + + A BM25 input feeds an analyzer, so it is stored as "" rather than null — + which means the physical column must be declared NOT NULL even though the + logical field is optional. Episode and user_profile cannot cover this + interaction, so it is asserted where it actually occurs. + """ + logical = schema_for(foresight_repo.schema) + assert logical.field("evidence_tokens").nullable is True + assert "evidence_tokens" in logical.bm25_fields + + physical = {f.name: f for f in foresight_repo._repo()._physical_fields()} # type: ignore[attr-defined] + assert physical["evidence_tokens"].nullable is False + assert physical["evidence_tokens"].enable_analyzer is True + # "evidence" is the same optional text without the analyzer attached, so + # it stays nullable — the NOT NULL above is caused by BM25, nothing else. + assert physical["evidence"].nullable is True diff --git a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py index 76990db7e..679925370 100644 --- a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py +++ b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py @@ -29,6 +29,7 @@ from typing import Any +from everos.infra.persistence.predicate import AnyOf, Comparison, IsNull, Predicate from everos.memory.cascade._backfill import ( NullBackfillPresenter, _backfill_table, @@ -59,9 +60,9 @@ class _RecordingRepo: assert on the exact ``{col: value}`` shape written back.""" def __init__(self) -> None: - self.updates: list[tuple[dict[str, Any], str]] = [] + self.updates: list[tuple[dict[str, Any], Predicate]] = [] - async def update(self, values: dict[str, Any], *, where: str) -> None: + async def update(self, values: dict[str, Any], *, where: Predicate) -> None: self.updates.append((values, where)) @@ -133,7 +134,7 @@ async def test_subject_only_failure_leaves_row_in_backlog() -> None: # The widened Episode scan filter would still match these rows # since their subject_vector remains NULL. - assert _null_filter(spec) == "vector IS NULL OR subject_vector IS NULL" + assert _null_filter(spec) == AnyOf((IsNull("vector"), IsNull("subject_vector"))) # Simulate a fresh scan encountering the residue: primary vector # is now populated (from the last write), subject_vector is None. @@ -200,11 +201,12 @@ async def test_orthogonal_partial_states_all_recover() -> None: assert result.rows_processed == 3 assert result.rows_failed == 0 - # Recover per-id updates by parsing the where clause - # (``id = 'xxx'``) — the repo double preserves call order and shape. + # Recover per-id updates from the neutral predicate tree. updates_by_id: dict[str, dict[str, Any]] = {} for values, where in repo.updates: - row_id = where.split("'")[1] + assert isinstance(where, Comparison) + assert where.field == "id" + row_id = str(where.value) updates_by_id[row_id] = values assert set(updates_by_id) == {"both", "subj_only", "prim_only"} @@ -222,14 +224,14 @@ def test_null_filter_widens_only_for_episode() -> None: ``subject_vector`` column, so referencing it would blow up).""" ep_repo = _RecordingRepo() ep_spec = _episode_spec(ep_repo) - assert _null_filter(ep_spec) == "vector IS NULL OR subject_vector IS NULL" + assert _null_filter(ep_spec) == AnyOf((IsNull("vector"), IsNull("subject_vector"))) fact_spec = _TableSpec( schema=_NonSubjectSchema, # type: ignore[arg-type] repo=_RecordingRepo(), # type: ignore[arg-type] text_of=lambda r: r["fact"], ) - assert _null_filter(fact_spec) == "vector IS NULL" + assert _null_filter(fact_spec) == IsNull("vector") def test_extract_row_skips_legitimate_null_subject() -> None: diff --git a/tests/unit/test_memory/test_cascade/test_backfill_table_specs.py b/tests/unit/test_memory/test_cascade/test_backfill_table_specs.py index d4ca1ea17..d76833bb0 100644 --- a/tests/unit/test_memory/test_cascade/test_backfill_table_specs.py +++ b/tests/unit/test_memory/test_cascade/test_backfill_table_specs.py @@ -18,7 +18,8 @@ import pytest -from everos.infra.persistence.lancedb import BUSINESS_SCHEMAS_WITH_VECTOR +from everos.infra.persistence.index import ALL_REPOS, Episode +from everos.infra.persistence.index.schema import schema_for from everos.memory.cascade import _backfill @@ -31,7 +32,11 @@ def test_table_specs_covers_business_schemas() -> None: tree: the two sets match at import time as well. """ spec_names = {spec.schema.TABLE_NAME for spec in _backfill._TABLE_SPECS} - schema_names = {schema.TABLE_NAME for schema in BUSINESS_SCHEMAS_WITH_VECTOR} + schema_names = { + repo.schema.TABLE_NAME + for repo in ALL_REPOS + if schema_for(repo.schema).vector_fields + } assert spec_names == schema_names @@ -50,19 +55,23 @@ def test_drift_scenario_actually_raises_at_import() -> None: """ import importlib from types import SimpleNamespace + from typing import ClassVar - import everos.infra.persistence.lancedb as lancedb_infra + import everos.infra.persistence.index as index_infra import everos.memory.cascade._backfill as backfill_mod - real_schemas = tuple(lancedb_infra.BUSINESS_SCHEMAS_WITH_VECTOR) - fake_schema = SimpleNamespace(TABLE_NAME="synthetic_drift_kind") - monkey_schemas = (*real_schemas, fake_schema) + class _SyntheticDriftSchema(Episode): + TABLE_NAME: ClassVar[str] = "synthetic_drift_kind" - original = lancedb_infra.BUSINESS_SCHEMAS_WITH_VECTOR - lancedb_infra.BUSINESS_SCHEMAS_WITH_VECTOR = monkey_schemas # type: ignore[misc] + fake_schema = _SyntheticDriftSchema + fake_repo = SimpleNamespace(schema=fake_schema) + monkey_repos = (*index_infra.ALL_REPOS, fake_repo) + + original = index_infra.ALL_REPOS + index_infra.ALL_REPOS = monkey_repos # type: ignore[misc] try: with pytest.raises(RuntimeError, match=r"synthetic_drift_kind|drift"): importlib.reload(backfill_mod) finally: - lancedb_infra.BUSINESS_SCHEMAS_WITH_VECTOR = original # type: ignore[misc] + index_infra.ALL_REPOS = original # type: ignore[misc] importlib.reload(backfill_mod) # restore module state diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py index 9f79cc562..c29782c60 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py @@ -23,7 +23,7 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot -from everos.infra.persistence.lancedb import AgentCase +from everos.infra.persistence.index import AgentCase, Comparison, Predicate from everos.infra.persistence.markdown import AgentCaseWriter from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.agent_case import AgentCaseHandler @@ -61,10 +61,11 @@ def __init__(self) -> None: self.deletes: list[str] = [] self.rows: list[AgentCase] = [] - async def find_where(self, where: str, *, limit: int = 100) -> list[AgentCase]: - prefix = "md_path = '" - if where.startswith(prefix): - md_path = where[len(prefix) :].rstrip("'") + async def find_where( + self, where: Predicate, *, limit: int = 100 + ) -> list[AgentCase]: + if isinstance(where, Comparison) and where.field == "md_path": + md_path = str(where.value) return [r for r in self.rows if r.md_path == md_path] return [] @@ -75,7 +76,7 @@ async def upsert(self, rows: list[AgentCase]) -> None: by_id[r.id] = r self.rows = list(by_id.values()) - async def delete(self, predicate: str) -> None: + async def delete(self, predicate: Predicate) -> None: self.deletes.append(predicate) async def delete_by_md_path(self, md_path: str) -> int: @@ -111,7 +112,7 @@ def no_embedder(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.fixture def fake_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeAgentCaseRepo: repo = _FakeAgentCaseRepo() - monkeypatch.setattr(AgentCaseHandler, "lance_repo", repo) + monkeypatch.setattr(AgentCaseHandler, "index_repo", repo) return repo diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py index 8ed467d45..290a7a673 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py @@ -18,7 +18,15 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot -from everos.infra.persistence.lancedb import AgentSkill +from everos.infra.persistence.index import ( + AgentSkill, + All, + Comparison, + Predicate, + all_of, + eq, + ne, +) from everos.infra.persistence.markdown import AgentSkillWriter from everos.memory.cascade.handlers import AgentSkillHandler, HandlerDeps @@ -72,22 +80,31 @@ async def delete_by_md_path(self, md_path: str) -> int: self.deletes.append(md_path) return 1 - async def find_where(self, predicate: str, *, limit: int) -> list[AgentSkill]: - """In-memory equivalent — handles only the - ``md_path = '...' AND id != '...'`` shape the handler emits.""" - if "md_path = " in predicate and "id != " in predicate: - md_lit = predicate.split("md_path = '")[1].split("'", 1)[0] - id_lit = predicate.split("id != '")[1].split("'", 1)[0] + async def find_where(self, predicate: Predicate, *, limit: int) -> list[AgentSkill]: + """In-memory equivalent for the handler's neutral predicate.""" + if isinstance(predicate, All): + values = { + child.field: child.value + for child in predicate.children + if isinstance(child, Comparison) + } + md_lit = values.get("md_path") + id_lit = values.get("id") return [ r for r in self.rows.values() if r.md_path == md_lit and r.id != id_lit ][:limit] raise NotImplementedError(f"fake repo doesn't handle {predicate!r}") - async def delete(self, predicate: str) -> None: + async def delete(self, predicate: Predicate) -> None: self.predicate_deletes.append(predicate) - if "md_path = " in predicate and "id != " in predicate: - md_lit = predicate.split("md_path = '")[1].split("'", 1)[0] - id_lit = predicate.split("id != '")[1].split("'", 1)[0] + if isinstance(predicate, All): + values = { + child.field: child.value + for child in predicate.children + if isinstance(child, Comparison) + } + md_lit = values.get("md_path") + id_lit = values.get("id") self.rows = { rid: row for rid, row in self.rows.items() @@ -233,7 +250,7 @@ async def test_renaming_skill_via_frontmatter_clears_old_row( assert list(fake_repo.rows.keys()) == ["a1_new_name"] # The sweep predicate references the *new* id with the same md_path. assert fake_repo.predicate_deletes == [ - f"md_path = '{md_path}' AND id != 'a1_new_name'" + all_of(eq("md_path", md_path), ne("id", "a1_new_name")) ] diff --git a/tests/unit/test_memory/test_cascade/test_handler_episode.py b/tests/unit/test_memory/test_cascade/test_handler_episode.py index a8af7fe1a..d423aa64d 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_episode.py +++ b/tests/unit/test_memory/test_cascade/test_handler_episode.py @@ -18,7 +18,7 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot -from everos.infra.persistence.lancedb import Episode +from everos.infra.persistence.index import Comparison, Episode, Predicate from everos.infra.persistence.markdown import EpisodeWriter from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.episode import EpisodeHandler @@ -58,11 +58,9 @@ def __init__(self) -> None: self.deletes: list[str] = [] self.rows: list[Episode] = [] - async def find_where(self, where: str, *, limit: int = 100) -> list[Episode]: - # Honour only the md_path = '...' filter the handler emits. - prefix = "md_path = '" - if where.startswith(prefix): - md_path = where[len(prefix) :].rstrip("'") + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[Episode]: + if isinstance(where, Comparison) and where.field == "md_path": + md_path = str(where.value) return [r for r in self.rows if r.md_path == md_path] return [] @@ -74,7 +72,7 @@ async def upsert(self, rows: list[Episode]) -> None: by_id[r.id] = r self.rows = list(by_id.values()) - async def delete(self, predicate: str) -> None: + async def delete(self, predicate: Predicate) -> None: self.deletes.append(predicate) async def delete_by_md_path(self, md_path: str) -> int: @@ -104,7 +102,7 @@ def stub_embedder(monkeypatch: pytest.MonkeyPatch) -> _StubEmbedder: @pytest.fixture def fake_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeEpisodeRepo: - """Swap the class-level ``lance_repo`` on EpisodeHandler. + """Swap the class-level ``index_repo`` on EpisodeHandler. After the BaseDailyLogHandler refactor, the repo binding is a ClassVar resolved at class-definition time; patching the module @@ -113,7 +111,7 @@ def fake_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeEpisodeRepo: from everos.memory.cascade.handlers.episode import EpisodeHandler repo = _FakeEpisodeRepo() - monkeypatch.setattr(EpisodeHandler, "lance_repo", repo) + monkeypatch.setattr(EpisodeHandler, "index_repo", repo) return repo diff --git a/tests/unit/test_memory/test_cascade/test_registry_knowledge.py b/tests/unit/test_memory/test_cascade/test_registry_knowledge.py index 3970250f7..f4dadd352 100644 --- a/tests/unit/test_memory/test_cascade/test_registry_knowledge.py +++ b/tests/unit/test_memory/test_cascade/test_registry_knowledge.py @@ -28,12 +28,12 @@ def test_match_topic_md(self) -> None: assert spec is not None assert spec.name == "knowledge_topic" - def test_knowledge_document_has_no_lance_schema(self) -> None: + def test_knowledge_document_has_no_index_schema(self) -> None: spec = next(k for k in KIND_REGISTRY if k.name == "knowledge_document") - assert spec.lance_schema is None - assert spec.lance_repo is None + assert spec.index_schema is None + assert spec.index_repo is None - def test_knowledge_topic_has_lance_schema(self) -> None: + def test_knowledge_topic_has_index_schema(self) -> None: spec = next(k for k in KIND_REGISTRY if k.name == "knowledge_topic") - assert spec.lance_schema is not None - assert spec.lance_repo is not None + assert spec.index_schema is not None + assert spec.index_repo is not None diff --git a/tests/unit/test_memory/test_cascade/test_worker.py b/tests/unit/test_memory/test_cascade/test_worker.py index f60fbbbe1..e7953c34a 100644 --- a/tests/unit/test_memory/test_cascade/test_worker.py +++ b/tests/unit/test_memory/test_cascade/test_worker.py @@ -335,17 +335,17 @@ async def rebuild_indexes(self) -> None: class _OkHandlerWithRepo(_OkHandler): - """OK handler exposing a fake ``lance_repo`` for scheduler tests.""" + """OK handler exposing a fake ``index_repo`` for scheduler tests.""" def __init__(self, repo: _FakeLanceRepo) -> None: super().__init__() - self.lance_repo = repo + self.index_repo = repo -async def test_schedule_optimize_noop_when_handler_has_no_lance_repo( +async def test_schedule_optimize_noop_when_handler_has_no_index_repo( patched_repo: _FakeRepo, ) -> None: - """Test stubs without ``lance_repo`` should not even register state.""" + """Test stubs without ``index_repo`` should not even register state.""" w = CascadeWorker( {"episode": _OkHandler()}, retry_backoff_seconds=0, @@ -515,7 +515,7 @@ async def prune(self, older_than: dt.timedelta) -> None: class _HandlerWithFailingRepo(_OkHandler): def __init__(self) -> None: super().__init__() - self.lance_repo = _FailingRepo() + self.index_repo = _FailingRepo() patched_repo.batch = [_Row(md_path="a.md")] w = CascadeWorker( diff --git a/tests/unit/test_memory/test_get/test_filters_adapter.py b/tests/unit/test_memory/test_get/test_filters_adapter.py index 4d6c48f4d..e5815dc73 100644 --- a/tests/unit/test_memory/test_get/test_filters_adapter.py +++ b/tests/unit/test_memory/test_get/test_filters_adapter.py @@ -4,62 +4,101 @@ :func:`everos.memory.search.compile_filters` — these tests pin the behaviour /get callers depend on: -* base clause shape (``owner_id = '...' AND owner_type = '...'``) -* flat multi-field → implicit ``AND`` +* base clause shape (owner / owner_type / app / project / deprecated_by) +* flat multi-field → implicit conjunction * reserved field (``owner_id`` / ``owner_type`` inside ``filters``) → :class:`FilterError` * unknown field → :class:`FilterError` * top-level ``AND`` / ``OR`` combinators are accepted (parity with ``/search`` — the wiki §附录 C restriction was dropped 2026-05-16) -* ``timestamp`` range (multi-op map) renders ``AND``-folded clauses -* ``sender_id`` is an array column → ``array_has(...)`` rendering +* ``timestamp`` range (multi-op map) folds into the conjunction +* ``sender_id`` is an array column → ``Contains`` nodes + +Assertions target the backend-neutral predicate tree; rendered syntax is an +adapter concern and is pinned in ``tests/unit/test_infra``. """ from __future__ import annotations import pytest +from everos.component.utils.datetime import from_iso_format, from_timestamp +from everos.infra.persistence.index import ( + All, + AnyOf, + Comparison, + Contains, + Predicate, + contains, + eq, + is_null, +) from everos.memory.get.filters_adapter import compile_filters_for_get from everos.memory.search import FilterError, FilterNode +def _clauses(predicate: Predicate) -> list[Predicate]: + """Flatten a compiled tree into its leaf clauses, descending into groups.""" + if isinstance(predicate, All | AnyOf): + return [leaf for child in predicate.children for leaf in _clauses(child)] + return [predicate] + + +def _groups(predicate: Predicate, kind: type[Predicate]) -> list[Predicate]: + """Every group node of ``kind`` in the tree, outermost first.""" + found: list[Predicate] = [predicate] if isinstance(predicate, kind) else [] + if isinstance(predicate, All | AnyOf): + for child in predicate.children: + found.extend(_groups(child, kind)) + return found + + +def _u1_filter(raw: dict[str, object]) -> Predicate: + node = FilterNode.model_validate(raw) + return compile_filters_for_get(node, owner_id="u1", owner_type="user") + + def test_no_filters_emits_base_clause() -> None: - """``filters=None`` → owner + app/project scope clauses AND-joined.""" + """``filters=None`` → owner + app/project scope clauses, AND-joined.""" where = compile_filters_for_get(None, owner_id="u1", owner_type="user") - assert where == ( - "owner_id = 'u1' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + assert where == All( + ( + eq("owner_id", "u1"), + eq("owner_type", "user"), + eq("app_id", "default"), + eq("project_id", "default"), + is_null("deprecated_by"), + ) ) def test_no_filters_agent_omits_deprecated_by() -> None: """Agent tables lack ``deprecated_by`` — clause must be absent.""" where = compile_filters_for_get(None, owner_id="bot", owner_type="agent") - assert "deprecated_by" not in where + assert is_null("deprecated_by") not in _clauses(where) -def test_owner_id_quote_is_escaped() -> None: - """SQL-standard double-quote escape on ``owner_id``.""" +def test_owner_id_is_carried_verbatim() -> None: + """The compiler does not pre-quote — escaping belongs to the adapter.""" where = compile_filters_for_get(None, owner_id="o'reilly", owner_type="user") - assert where == ( - "owner_id = 'o''reilly' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" - ) + assert eq("owner_id", "o'reilly") in _clauses(where) def test_flat_multi_field_renders_implicit_and() -> None: - """Multiple top-level fields → implicit ``AND`` between predicates.""" - node = FilterNode.model_validate({"session_id": "sess_a", "parent_id": "mc_x"}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - # Field iteration order follows insertion order, so both are present. - assert "owner_id = 'u1'" in where - assert "owner_type = 'user'" in where - assert "session_id = 'sess_a'" in where - assert "parent_id = 'mc_x'" in where - # 5 base scope clauses + 2 filter fields = 7 clauses → 6 ' AND ' joins. - assert where.count(" AND ") == 6 + """Multiple top-level fields → implicit conjunction with the base clauses.""" + where = _u1_filter({"session_id": "sess_a", "parent_id": "mc_x"}) + # 5 base scope clauses + 2 filter fields, flattened into one conjunction. + assert where == All( + ( + eq("owner_id", "u1"), + eq("owner_type", "user"), + eq("app_id", "default"), + eq("project_id", "default"), + is_null("deprecated_by"), + eq("session_id", "sess_a"), + eq("parent_id", "mc_x"), + ) + ) def test_reserved_owner_id_in_filters_raises() -> None: @@ -83,83 +122,74 @@ def test_unsupported_field_raises() -> None: compile_filters_for_get(node, owner_id="u1", owner_type="user") -def test_timestamp_range_renders_and_folded() -> None: - """Multi-op map on one field folds with ``AND`` (reused from /search).""" - node = FilterNode.model_validate( - {"timestamp": {"gte": 1704067200000, "lt": 1735689600000}} - ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "timestamp >= TIMESTAMP '" in where - assert "timestamp < TIMESTAMP '" in where - # The two clauses are AND-joined inside one parenthesised group. - assert "(timestamp >= TIMESTAMP" in where - assert " AND timestamp < TIMESTAMP" in where +def test_timestamp_range_folds_into_conjunction() -> None: + """Multi-op map on one field folds with AND (reused from /search).""" + where = _u1_filter({"timestamp": {"gte": 1704067200000, "lt": 1735689600000}}) + clauses = _clauses(where) + assert Comparison("timestamp", "gte", from_timestamp(1704067200000)) in clauses + assert Comparison("timestamp", "lt", from_timestamp(1735689600000)) in clauses + assert _groups(where, AnyOf) == [] -def test_sender_id_in_list_renders_array_has() -> None: - """``sender_id`` is an array column — ``in`` → ``array_has(...) OR ...``.""" - node = FilterNode.model_validate({"sender_id": {"in": ["alice", "bob"]}}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "array_has(sender_ids, 'alice')" in where - assert "array_has(sender_ids, 'bob')" in where +def test_sender_id_in_list_becomes_contains() -> None: + """``sender_id`` is an array column — ``in`` → OR of ``Contains``.""" + where = _u1_filter({"sender_id": {"in": ["alice", "bob"]}}) + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + assert disjunctions[0].children == ( # type: ignore[attr-defined] + Contains("sender_ids", "alice"), + Contains("sender_ids", "bob"), + ) -def test_sender_id_eq_shorthand_renders_array_has() -> None: - """Equality shorthand on an array column → single ``array_has``.""" - node = FilterNode.model_validate({"sender_id": "alice"}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "array_has(sender_ids, 'alice')" in where +def test_sender_id_eq_shorthand_becomes_contains() -> None: + """Equality shorthand on an array column → a single ``Contains``.""" + where = _u1_filter({"sender_id": "alice"}) + assert contains("sender_ids", "alice") in _clauses(where) -def test_parent_id_eq_shorthand_renders_scalar_eq() -> None: - """``parent_id`` is a scalar string column → plain ``=``.""" - node = FilterNode.model_validate({"parent_id": "mc_42"}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "parent_id = 'mc_42'" in where +def test_parent_id_eq_shorthand_stays_scalar_eq() -> None: + """``parent_id`` is a scalar string column → plain equality.""" + where = _u1_filter({"parent_id": "mc_42"}) + assert eq("parent_id", "mc_42") in _clauses(where) -def test_top_level_and_renders_grouped_clause() -> None: - """``AND`` combinator compiles like /search — parens-grouped fragments.""" - node = FilterNode.model_validate( - {"AND": [{"session_id": "sess_a"}, {"parent_id": "mc_x"}]} - ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - # Base clause is always first; combinator output appended. - assert where.startswith("owner_id = 'u1' AND owner_type = 'user' AND ") - assert "session_id = 'sess_a'" in where - assert "parent_id = 'mc_x'" in where +def test_top_level_and_folds_into_conjunction() -> None: + """``AND`` combinator compiles like /search.""" + where = _u1_filter({"AND": [{"session_id": "sess_a"}, {"parent_id": "mc_x"}]}) + clauses = _clauses(where) + assert eq("owner_id", "u1") in clauses + assert eq("owner_type", "user") in clauses + assert eq("session_id", "sess_a") in clauses + assert eq("parent_id", "mc_x") in clauses + assert _groups(where, AnyOf) == [] -def test_top_level_or_renders_grouped_clause() -> None: - """``OR`` combinator emits parens-grouped ``OR`` between sibling preds.""" - node = FilterNode.model_validate( - {"OR": [{"session_id": "sess_a"}, {"session_id": "sess_b"}]} - ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "session_id = 'sess_a'" in where - assert "session_id = 'sess_b'" in where - assert " OR " in where +def test_top_level_or_emits_disjunction() -> None: + """``OR`` combinator emits a disjunction between sibling predicates.""" + where = _u1_filter({"OR": [{"session_id": "sess_a"}, {"session_id": "sess_b"}]}) + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + branches = [_clauses(child) for child in disjunctions[0].children] # type: ignore[attr-defined] + assert branches == [[eq("session_id", "sess_a")], [eq("session_id", "sess_b")]] -def test_ne_operator_renders_not_equal() -> None: - """``ne`` op compiles to ``!=`` on str fields.""" - node = FilterNode.model_validate({"session_id": {"ne": "sess_internal"}}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "session_id != 'sess_internal'" in where +def test_ne_operator_kept_as_ne() -> None: + """``ne`` op compiles to a ``ne`` comparison on str fields.""" + where = _u1_filter({"session_id": {"ne": "sess_internal"}}) + assert Comparison("session_id", "ne", "sess_internal") in _clauses(where) -def test_timestamp_iso_string_renders_literal() -> None: - """ISO 8601 string is accepted as a timestamp literal (alongside epoch ms).""" - node = FilterNode.model_validate( - {"timestamp": {"gte": "2026-01-04T00:00:00+00:00"}} - ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "timestamp >= TIMESTAMP '2026-01-04T00:00:00+00:00'" in where +def test_timestamp_iso_string_normalized_to_datetime() -> None: + """ISO 8601 strings are accepted alongside epoch ms and normalized.""" + where = _u1_filter({"timestamp": {"gte": "2026-01-04T00:00:00+00:00"}}) + expected = from_iso_format("2026-01-04T00:00:00+00:00") + assert Comparison("timestamp", "gte", expected) in _clauses(where) def test_nested_and_inside_or() -> None: """``AND`` nested inside ``OR`` — combinators compose recursively.""" - node = FilterNode.model_validate( + where = _u1_filter( { "OR": [ {"AND": [{"session_id": "sess_a"}, {"parent_id": "mc_x"}]}, @@ -167,12 +197,13 @@ def test_nested_and_inside_or() -> None: ] } ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "session_id = 'sess_a'" in where - assert "parent_id = 'mc_x'" in where - assert "session_id = 'sess_b'" in where - assert " OR " in where - assert " AND " in where + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + branches = [_clauses(child) for child in disjunctions[0].children] # type: ignore[attr-defined] + assert branches == [ + [eq("session_id", "sess_a"), eq("parent_id", "mc_x")], + [eq("session_id", "sess_b")], + ] # ── Malformed value shapes ────────────────────────────────────────────── diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index 35cec6b79..6efa3d55c 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -254,9 +254,12 @@ async def test_episodic_memory_passes_where_and_sort_to_repo( filters=FilterNode.model_validate({"session_id": "sess_a"}), ) await mgr.get(req) - assert "owner_id = 'u1'" in ep.last.where - assert "owner_type = 'user'" in ep.last.where - assert "session_id = 'sess_a'" in ep.last.where + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(ep.last.where) + assert "owner_id = 'u1'" in rendered + assert "owner_type = 'user'" in rendered + assert "session_id = 'sess_a'" in rendered assert ep.last.sort_by == "timestamp" assert ep.last.descending is False # asc assert ep.last.page == 2 diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index 808c2a382..23e854601 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -1,39 +1,89 @@ -"""Unit tests for the Filters DSL compiler.""" +"""Unit tests for the Filters DSL compiler. + +The compiler's output is a backend-neutral :class:`Predicate` tree, so these +assertions are made against that tree rather than against any adapter's +rendered syntax. Quoting, literal formats and array-containment syntax belong +to the adapters and are pinned in ``tests/unit/test_infra`` instead. +""" from __future__ import annotations +import datetime as dt + import pytest +from everos.component.utils.datetime import from_timestamp +from everos.infra.persistence.index import ( + All, + AnyOf, + Comparison, + Contains, + Predicate, + contains, + eq, + is_null, + one_of, +) from everos.memory.search import ( FilterError, FilterNode, compile_filters, ) +_BASE_USER = ( + eq("owner_id", "alice"), + eq("owner_type", "user"), + eq("app_id", "default"), + eq("project_id", "default"), + is_null("deprecated_by"), +) + + +def _clauses(predicate: Predicate) -> list[Predicate]: + """Flatten a compiled tree into its leaf clauses, descending into groups.""" + if isinstance(predicate, All | AnyOf): + return [leaf for child in predicate.children for leaf in _clauses(child)] + return [predicate] + + +def _groups(predicate: Predicate, kind: type[Predicate]) -> list[Predicate]: + """Every group node of ``kind`` in the tree, outermost first.""" + found: list[Predicate] = [predicate] if isinstance(predicate, kind) else [] + if isinstance(predicate, All | AnyOf): + for child in predicate.children: + found.extend(_groups(child, kind)) + return found + + +def _user_filter(raw: dict[str, object]) -> Predicate: + node = FilterNode.model_validate(raw) + return compile_filters(node, owner_id="alice", owner_type="user") + + # ── Base injection ─────────────────────────────────────────────────────── def test_no_filters_emits_base_clause() -> None: where = compile_filters(None, owner_id="alice", owner_type="user") - assert where == ( - "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" - ) + assert where == All(_BASE_USER) def test_no_filters_agent_omits_deprecated_by() -> None: where = compile_filters(None, owner_id="bot_42", owner_type="agent") - assert "deprecated_by" not in where - assert where == ( - "owner_id = 'bot_42' AND owner_type = 'agent' " - "AND app_id = 'default' AND project_id = 'default'" + assert where == All( + ( + eq("owner_id", "bot_42"), + eq("owner_type", "agent"), + eq("app_id", "default"), + eq("project_id", "default"), + ) ) + assert is_null("deprecated_by") not in _clauses(where) def test_owner_type_agent_pinned() -> None: where = compile_filters(None, owner_id="alice", owner_type="agent") - assert "owner_type = 'agent'" in where + assert eq("owner_type", "agent") in _clauses(where) def test_app_project_scope_pinned() -> None: @@ -44,55 +94,58 @@ def test_app_project_scope_pinned() -> None: app_id="claude_code", project_id="oss", ) - assert "app_id = 'claude_code'" in where - assert "project_id = 'oss'" in where + assert eq("app_id", "claude_code") in _clauses(where) + assert eq("project_id", "oss") in _clauses(where) -def test_owner_id_with_quote_is_escaped() -> None: +def test_owner_id_is_carried_verbatim() -> None: + """The compiler must not pre-escape — quoting is the adapter's job. + + Escaping itself is asserted against the LanceDB renderer in + ``tests/unit/test_infra/test_index_contract.py``. + """ where = compile_filters(None, owner_id="al'ice", owner_type="user") - assert "owner_id = 'al''ice'" in where + assert eq("owner_id", "al'ice") in _clauses(where) # ── Equality / shorthand ──────────────────────────────────────────────── def test_flat_equality_shorthand() -> None: - node = FilterNode(session_id="sess_a") # type: ignore[call-arg] - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "session_id = 'sess_a'" in where + where = _user_filter({"session_id": "sess_a"}) + assert eq("session_id", "sess_a") in _clauses(where) def test_multiple_flat_fields_join_with_and() -> None: - node = FilterNode.model_validate({"session_id": "sess_a", "parent_type": "memcell"}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "session_id = 'sess_a'" in where - assert "parent_type = 'memcell'" in where + where = _user_filter({"session_id": "sess_a", "parent_type": "memcell"}) + assert where == All( + (*_BASE_USER, eq("session_id", "sess_a"), eq("parent_type", "memcell")) + ) # ── Operators ─────────────────────────────────────────────────────────── -def test_timestamp_gte_renders_timestamp_literal() -> None: - node = FilterNode.model_validate({"timestamp": {"gte": 1704067200000}}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "timestamp >= TIMESTAMP '" in where +def test_timestamp_gte_normalized_to_datetime() -> None: + """Epoch ms become a real ``datetime`` in the AST, not a rendered literal.""" + where = _user_filter({"timestamp": {"gte": 1704067200000}}) + clause = Comparison("timestamp", "gte", from_timestamp(1704067200000)) + assert clause in _clauses(where) + assert isinstance(clause.value, dt.datetime) def test_timestamp_range_folds_with_and() -> None: - node = FilterNode.model_validate( - {"timestamp": {"gte": 1704067200000, "lt": 1740614399000}} - ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "timestamp >= TIMESTAMP '" in where - assert "timestamp < TIMESTAMP '" in where - # Operators on the same field are wrapped in a single group. - assert " AND " in where + where = _user_filter({"timestamp": {"gte": 1704067200000, "lt": 1740614399000}}) + clauses = _clauses(where) + assert Comparison("timestamp", "gte", from_timestamp(1704067200000)) in clauses + assert Comparison("timestamp", "lt", from_timestamp(1740614399000)) in clauses + # Operators on the same field sit in the conjunction, not a disjunction. + assert _groups(where, AnyOf) == [] def test_in_operator_string_field() -> None: - node = FilterNode.model_validate({"parent_type": {"in": ["memcell", "episode"]}}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "parent_type IN ('memcell', 'episode')" in where + where = _user_filter({"parent_type": {"in": ["memcell", "episode"]}}) + assert one_of("parent_type", ["memcell", "episode"]) in _clauses(where) def test_in_operator_requires_non_empty_list() -> None: @@ -111,7 +164,7 @@ def test_invalid_operator_rejected() -> None: def test_and_combinator() -> None: - node = FilterNode.model_validate( + where = _user_filter( { "AND": [ {"timestamp": {"gte": 1704067200000}}, @@ -119,29 +172,24 @@ def test_and_combinator() -> None: ] } ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "timestamp >= TIMESTAMP '" in where - assert "timestamp < TIMESTAMP '" in where - assert " AND " in where + clauses = _clauses(where) + assert Comparison("timestamp", "gte", from_timestamp(1704067200000)) in clauses + assert Comparison("timestamp", "lt", from_timestamp(1740614399000)) in clauses + assert _groups(where, AnyOf) == [] def test_or_combinator() -> None: - node = FilterNode.model_validate( - { - "OR": [ - {"parent_type": "memcell"}, - {"parent_type": "episode"}, - ] - } + where = _user_filter( + {"OR": [{"parent_type": "memcell"}, {"parent_type": "episode"}]} ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert " OR " in where - assert "parent_type = 'memcell'" in where - assert "parent_type = 'episode'" in where + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + branches = [_clauses(child) for child in disjunctions[0].children] # type: ignore[attr-defined] + assert branches == [[eq("parent_type", "memcell")], [eq("parent_type", "episode")]] def test_nested_and_inside_or() -> None: - node = FilterNode.model_validate( + where = _user_filter( { "OR": [ {"AND": [{"parent_type": "memcell"}, {"session_id": "sa"}]}, @@ -149,41 +197,38 @@ def test_nested_and_inside_or() -> None: ] } ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "parent_type = 'memcell'" in where - assert "session_id = 'sa'" in where - assert "parent_type = 'episode'" in where - assert " OR " in where - assert " AND " in where + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + branches = [_clauses(child) for child in disjunctions[0].children] # type: ignore[attr-defined] + assert branches == [ + [eq("parent_type", "memcell"), eq("session_id", "sa")], + [eq("parent_type", "episode")], + ] def test_flat_field_alongside_and_combinator() -> None: - node = FilterNode.model_validate( - { - "session_id": "sess_a", - "AND": [{"timestamp": {"gte": 1}}], - } - ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "session_id = 'sess_a'" in where - assert "timestamp >= TIMESTAMP '" in where + where = _user_filter({"session_id": "sess_a", "AND": [{"timestamp": {"gte": 1}}]}) + clauses = _clauses(where) + assert eq("session_id", "sess_a") in clauses + assert Comparison("timestamp", "gte", from_timestamp(1)) in clauses # ── Array field (sender_id → sender_ids) ──────────────────────────────── -def test_sender_id_eq_uses_array_has() -> None: - node = FilterNode.model_validate({"sender_id": "u_jason"}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "array_has(sender_ids, 'u_jason')" in where +def test_sender_id_eq_becomes_contains() -> None: + where = _user_filter({"sender_id": "u_jason"}) + assert contains("sender_ids", "u_jason") in _clauses(where) -def test_sender_id_in_expands_to_or_array_has() -> None: - node = FilterNode.model_validate({"sender_id": {"in": ["u_a", "u_b"]}}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "array_has(sender_ids, 'u_a')" in where - assert "array_has(sender_ids, 'u_b')" in where - assert " OR " in where +def test_sender_id_in_expands_to_or_of_contains() -> None: + where = _user_filter({"sender_id": {"in": ["u_a", "u_b"]}}) + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + assert disjunctions[0].children == ( # type: ignore[attr-defined] + Contains("sender_ids", "u_a"), + Contains("sender_ids", "u_b"), + ) def test_sender_id_gt_rejected() -> None: @@ -213,10 +258,10 @@ def test_owner_type_in_filters_rejected() -> None: compile_filters(node, owner_id="alice", owner_type="user") -def test_string_with_single_quote_escaped() -> None: - node = FilterNode.model_validate({"session_id": "ses's"}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "session_id = 'ses''s'" in where +def test_string_with_single_quote_is_carried_verbatim() -> None: + """Filter values reach the AST unmodified; the adapter does the quoting.""" + where = _user_filter({"session_id": "ses's"}) + assert eq("session_id", "ses's") in _clauses(where) def test_timestamp_string_with_quote_rejected() -> None: @@ -248,21 +293,17 @@ def test_empty_and_array_skips_combinator() -> None: """Empty AND/OR arrays compile to no clauses — only the base remains.""" node = FilterNode.model_validate({"AND": []}) where = compile_filters(node, owner_id="alice", owner_type="user") - assert where == ( - "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" - ) + assert where == compile_filters(None, owner_id="alice", owner_type="user") # ── Deprecated exclusion ────────────────────────────────────────────── def test_compile_filters_excludes_deprecated_by_for_user() -> None: - result = compile_filters(None, owner_id="u_a", owner_type="user") - assert "deprecated_by IS NULL" in result + where = compile_filters(None, owner_id="u_a", owner_type="user") + assert is_null("deprecated_by") in _clauses(where) def test_compile_filters_omits_deprecated_by_for_agent() -> None: - result = compile_filters(None, owner_id="agent_1", owner_type="agent") - assert "deprecated_by" not in result + where = compile_filters(None, owner_id="agent_1", owner_type="agent") + assert is_null("deprecated_by") not in _clauses(where) diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index 73dd66611..f2f16da2b 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -393,8 +393,11 @@ async def test_user_keyword_filters_compile_pinned_owner() -> None: ) await mgr.search(_user_req()) assert recaller.last_where is not None - assert "owner_id = 'alice'" in recaller.last_where - assert "owner_type = 'user'" in recaller.last_where + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(recaller.last_where) + assert "owner_id = 'alice'" in rendered + assert "owner_type = 'user'" in rendered def _atomic_fact_row(fid: str, *, parent_id: str, score: float) -> Candidate: diff --git a/tests/unit/test_memory/test_search/test_recall_agent_skill.py b/tests/unit/test_memory/test_search/test_recall_agent_skill.py index 9239fa34a..661e58a0b 100644 --- a/tests/unit/test_memory/test_search/test_recall_agent_skill.py +++ b/tests/unit/test_memory/test_search/test_recall_agent_skill.py @@ -26,6 +26,7 @@ agent_skill_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall.agent_skill import AgentSkillRecaller from everos.memory.search.recall.base import RecallerDeps @@ -76,7 +77,7 @@ def _recaller() -> AgentSkillRecaller: return AgentSkillRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) -_OWNER_WHERE = "owner_id = 'agt' AND owner_type = 'agent'" +_OWNER_WHERE = all_of(eq("owner_id", "agt"), eq("owner_type", "agent")) async def test_fetch_by_case_ids_matches_any_lineage_case() -> None: diff --git a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py index f2e80b88e..e3d5dfd9e 100644 --- a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py +++ b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py @@ -29,6 +29,7 @@ atomic_fact_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall.atomic_fact import AtomicFactRecaller from everos.memory.search.recall.base import RecallerDeps @@ -83,6 +84,9 @@ def _recaller() -> AtomicFactRecaller: return AtomicFactRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) +_ALICE_WHERE = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) + + async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: """Two episodes sharing one memcell both see the same fact pool. @@ -104,8 +108,9 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: "alice_ep_b": ["mc_shared"], "alice_ep_c": ["mc_other"], } - where = "owner_id = 'alice' AND owner_type = 'user'" - out = await _recaller().facts_for_episodes(ep_to_parents, where, per_episode=10) + out = await _recaller().facts_for_episodes( + ep_to_parents, _ALICE_WHERE, per_episode=10 + ) assert sorted(out.keys()) == ["alice_ep_a", "alice_ep_b", "alice_ep_c"] assert sorted(f.id for f in out["alice_ep_a"]) == ["alice_af_1", "alice_af_2"] @@ -120,9 +125,7 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: async def test_facts_for_episodes_returns_empty_for_no_episodes() -> None: - out: dict = await _recaller().facts_for_episodes( - {}, "owner_id = 'alice'", per_episode=10 - ) + out: dict = await _recaller().facts_for_episodes({}, _ALICE_WHERE, per_episode=10) assert out == {} @@ -134,7 +137,7 @@ async def test_facts_for_episodes_skips_unknown_memcells() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"], "alice_ep_b": ["mc_missing"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert "alice_ep_a" in out @@ -163,7 +166,7 @@ async def test_facts_for_episodes_filters_by_where_clause() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert [f.id for f in out["alice_ep_a"]] == ["alice_af_1"] @@ -183,7 +186,7 @@ async def test_facts_for_episodes_drops_empty_parent_ids() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": [""]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert out == {} @@ -231,7 +234,7 @@ async def test_facts_for_episodes_assigns_real_cosine_score_with_query_vector() out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_shared"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, query_vector=_unit_vector(0), ) @@ -259,7 +262,7 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, # no query_vector ) @@ -287,7 +290,7 @@ async def test_facts_for_episodes_dual_parent_id_finds_both_eras() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["ep_entry_1", "mc_1"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) @@ -315,7 +318,7 @@ async def test_facts_for_episodes_multiple_parent_ids_dedup_across_episodes() -> "alice_ep_a": ["ep_entry_1", "mc_shared"], "alice_ep_b": ["ep_entry_2", "mc_shared"], }, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) diff --git a/tests/unit/test_memory/test_search/test_recall_episode.py b/tests/unit/test_memory/test_search/test_recall_episode.py index 4549322d8..6f853a303 100644 --- a/tests/unit/test_memory/test_search/test_recall_episode.py +++ b/tests/unit/test_memory/test_search/test_recall_episode.py @@ -8,6 +8,7 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.predicate import eq from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.episode import EpisodeRecaller @@ -42,6 +43,18 @@ def _mock_table(rows: list[dict[str, Any]]) -> MagicMock: return tbl +def _mock_records(rows: list[dict[str, Any]]) -> list[MagicMock]: + records: list[MagicMock] = [] + for row in rows: + record = MagicMock() + record.model_dump.return_value = row + records.append(record) + return records + + +_ALICE_WHERE = eq("owner_id", "alice") + + @pytest.fixture() def recaller() -> EpisodeRecaller: tok = MagicMock(spec=Tokenizer) @@ -58,11 +71,11 @@ async def test_fetch_all_for_owner_returns_entry_id_keyed_candidates( _make_row("ep_2", "mc_2"), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=_mock_records(rows), ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 2 ids = {c.id for c in result} @@ -75,11 +88,11 @@ async def test_fetch_all_for_owner_stores_episode_id_in_metadata( """metadata['episode_id'] carries the real LanceDB episode id for final shaping.""" rows = [_make_row("ep_abc", "mc_xyz")] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=_mock_records(rows), ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert result[0].metadata["episode_id"] == "ep_abc" assert result[0].metadata["parent_id"] == "mc_xyz" @@ -104,11 +117,11 @@ async def test_fetch_all_for_owner_skips_rows_without_entry_id( }, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=_mock_records(rows), ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert result == [] @@ -131,11 +144,11 @@ async def test_fetch_all_for_owner_merged_episode_uses_entry_id( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=_mock_records(rows), ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 1 assert result[0].id == "entry_xyz", "merged episode id must be entry_id" @@ -156,11 +169,11 @@ async def test_fetch_all_for_owner_mixed_regular_and_merged( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=_mock_records(rows), ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 2 ids = {c.id for c in result} @@ -179,16 +192,12 @@ async def test_fetch_by_entry_ids_returns_candidates( entry_id="entry_xyz", ), ] - mock_tbl = MagicMock() - mock_tbl.query.return_value.where.return_value.limit.return_value.to_list = ( - AsyncMock(return_value=rows) - ) with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=mock_tbl, + return_value=rows, ): - result = await recaller.fetch_by_entry_ids(["entry_xyz"], "owner_id = 'alice'") + result = await recaller.fetch_by_entry_ids(["entry_xyz"], _ALICE_WHERE) assert len(result) == 1 assert result[0].id == "ep_merged" @@ -198,7 +207,7 @@ async def test_fetch_by_entry_ids_empty_input_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty entry_ids list short-circuits without querying.""" - result = await recaller.fetch_by_entry_ids([], "owner_id = 'alice'") + result = await recaller.fetch_by_entry_ids([], _ALICE_WHERE) assert result == [] @@ -226,13 +235,11 @@ async def test_sparse_recall_as_child_injects_parent_id( {**_make_row("ep_1", "mc_1", entry_id="entry_1"), "_score": 1.0}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table(rows), + return_value=rows, ): - result = await recaller.sparse_recall_as_child( - "hello", "owner_id = 'alice'", limit=10 - ) + result = await recaller.sparse_recall_as_child("hello", _ALICE_WHERE, limit=10) assert len(result) == 1 assert result[0].metadata["parent_id"] == "entry_1" @@ -256,13 +263,11 @@ async def test_sparse_recall_as_child_falls_back_to_id_when_no_entry_id( "_score": 0.5, } with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table([row]), + return_value=[row], ): - result = await recaller.sparse_recall_as_child( - "hello", "owner_id = 'alice'", limit=10 - ) + result = await recaller.sparse_recall_as_child("hello", _ALICE_WHERE, limit=10) assert len(result) == 1 cand = result[0] @@ -274,7 +279,7 @@ async def test_sparse_recall_as_child_empty_query_returns_empty( ) -> None: """Empty query token list short-circuits; no table call needed.""" recaller._deps.tokenizer.tokenize.return_value = [] - result = await recaller.sparse_recall_as_child("", "owner_id = 'alice'", limit=10) + result = await recaller.sparse_recall_as_child("", _ALICE_WHERE, limit=10) assert result == [] @@ -286,12 +291,12 @@ async def test_dense_recall_as_child_injects_parent_id( {**_make_row("ep_3", "mc_3", entry_id="entry_3"), "_distance": 0.1}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.dense_search", new_callable=AsyncMock, - return_value=_mock_ann_table(rows), + return_value=rows, ): result = await recaller.dense_recall_as_child( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 @@ -302,7 +307,7 @@ async def test_dense_recall_as_child_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_as_child([], "owner_id = 'alice'", limit=10) + result = await recaller.dense_recall_as_child([], _ALICE_WHERE, limit=10) assert result == [] @@ -326,24 +331,30 @@ async def test_dense_recall_subject_returns_subject_vector_source( {**_make_row("ep_s1", "mc_s1", entry_id="entry_s1"), "_distance": 0.2}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.dense_search", new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), - ): + return_value=rows, + ) as dense_search: result = await recaller.dense_recall_subject( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 assert result[0].source == "vector" assert result[0].score == pytest.approx(0.8) + dense_search.assert_awaited_once_with( + [0.1] * 1024, + _ALICE_WHERE, + limit=10, + vector_field="subject_vector", + ) async def test_dense_recall_subject_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_subject([], "owner_id = 'alice'", limit=10) + result = await recaller.dense_recall_subject([], _ALICE_WHERE, limit=10) assert result == [] @@ -355,12 +366,12 @@ async def test_dense_recall_subject_as_child_injects_parent_id( {**_make_row("ep_s2", "mc_s2", entry_id="entry_s2"), "_distance": 0.15}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.dense_search", new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), + return_value=rows, ): result = await recaller.dense_recall_subject_as_child( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 @@ -372,7 +383,5 @@ async def test_dense_recall_subject_as_child_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_subject_as_child( - [], "owner_id = 'alice'", limit=10 - ) + result = await recaller.dense_recall_subject_as_child([], _ALICE_WHERE, limit=10) assert result == [] diff --git a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py index d40d8a41e..ea855584d 100644 --- a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py +++ b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py @@ -1,11 +1,11 @@ """Unit tests for ``KnowledgeTopicRecaller``. -Verifies dual-column BM25 + cosine ANN recall, using ``unittest.mock`` -to patch ``get_table`` so no real LanceDB connection is needed. +Verifies dual-column BM25 delegation + cosine ANN recall, using +``unittest.mock`` to patch the backend-neutral index repo. White-box surfaces touched: - - ``everos.memory.search.recall.knowledge_topic.get_table`` (patched) - - ``KnowledgeTopicRecaller.sparse_recall`` — queries both BM25 columns + - ``everos.memory.search.recall.knowledge_topic.knowledge_topic_repo`` (patched) + - ``KnowledgeTopicRecaller.sparse_recall`` — passes both BM25 columns - ``KnowledgeTopicRecaller.dense_recall`` — cosine ANN with distance→score """ @@ -17,6 +17,7 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.index import KnowledgeTopic, all_of, eq from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.knowledge_topic import KnowledgeTopicRecaller @@ -58,47 +59,12 @@ def _make_row( return row -def _mock_bm25_table( - summary_rows: list[dict[str, Any]], - content_rows: list[dict[str, Any]], -) -> MagicMock: - """Build a table mock whose BM25 results differ per column. - - The first ``nearest_to_text`` call (summary_tokens) returns - ``summary_rows``; the second (content_tokens) returns ``content_rows``. - ``asyncio.gather`` fires both concurrently, so we use ``side_effect`` - on the chain rather than recording call order. - """ - summary_chain = MagicMock() - summary_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=summary_rows - ) - - content_chain = MagicMock() - content_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=content_rows - ) - - tbl = MagicMock() - tbl.query.return_value.nearest_to_text.side_effect = [summary_chain, content_chain] - return tbl - - -def _mock_ann_table(rows: list[dict[str, Any]]) -> MagicMock: - """Build a table mock for ANN (dense) queries.""" - tbl = MagicMock() - ann = tbl.query.return_value.nearest_to.return_value - chain = ann.distance_type.return_value.where.return_value.limit.return_value - chain.to_list = AsyncMock(return_value=rows) - return tbl - - @pytest.fixture() def recaller() -> KnowledgeTopicRecaller: return KnowledgeTopicRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) -_WHERE = "app_id = 'app' AND project_id = 'proj'" +_WHERE = all_of(eq("app_id", "app"), eq("project_id", "proj")) # --------------------------------------------------------------------------- @@ -109,16 +75,20 @@ def recaller() -> KnowledgeTopicRecaller: async def test_sparse_recall_queries_both_columns( recaller: KnowledgeTopicRecaller, ) -> None: - """``nearest_to_text`` must be called once per BM25 column.""" - tbl = _mock_bm25_table( - summary_rows=[_make_row("t1", score=0.9)], - content_rows=[_make_row("t2", score=0.7)], - ) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + """Both BM25 columns must be delegated to the index repo.""" + rows = [_make_row("t1", score=0.9), _make_row("t2", score=0.7)] + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ) as mock_sparse: result = await recaller.sparse_recall("topic query", _WHERE, limit=10) - # nearest_to_text called twice (once per column) - assert tbl.query.return_value.nearest_to_text.call_count == 2 + mock_sparse.assert_awaited_once() + assert list(mock_sparse.await_args.args[0]) == ["topic", "query"] + assert mock_sparse.await_args.args[1] == _WHERE + assert mock_sparse.await_args.kwargs["columns"] == KnowledgeTopic.BM25_FIELDS + assert mock_sparse.await_args.kwargs["limit"] == 10 ids = {c.id for c in result} assert ids == {"t1", "t2"} @@ -126,13 +96,14 @@ async def test_sparse_recall_queries_both_columns( async def test_sparse_recall_merges_by_max_score( recaller: KnowledgeTopicRecaller, ) -> None: - """When the same id appears in both columns, keep the higher score.""" + """Scores returned by the repo are preserved on keyword candidates.""" shared_id = "topic_shared" - summary_rows = [_make_row(shared_id, score=0.5)] - content_rows = [_make_row(shared_id, score=0.9)] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=[_make_row(shared_id, score=0.9)], + ): result = await recaller.sparse_recall("overlap", _WHERE, limit=10) assert len(result) == 1 @@ -144,16 +115,16 @@ async def test_sparse_recall_merges_by_max_score( async def test_sparse_recall_returns_sorted_by_score( recaller: KnowledgeTopicRecaller, ) -> None: - """Merged results must be sorted descending by score, truncated to limit.""" - summary_rows = [ - _make_row("a", score=0.3), + """The recaller preserves repo ordering and maps rows to candidates.""" + rows = [ _make_row("b", score=0.8), - ] - content_rows = [ _make_row("c", score=0.6), ] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.sparse_recall("query", _WHERE, limit=2) assert len(result) == 2 @@ -164,16 +135,19 @@ async def test_sparse_recall_returns_sorted_by_score( async def test_sparse_recall_empty_query_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty tokenisation short-circuits — no LanceDB query is issued.""" + """Empty tokenisation short-circuits — no repo query is issued.""" tok = MagicMock(spec=Tokenizer) tok.tokenize.return_value = [] r = KnowledgeTopicRecaller(RecallerDeps(tokenizer=tok)) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + ) as mock_sparse: result = await r.sparse_recall("", _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_sparse.assert_not_called() # --------------------------------------------------------------------------- @@ -189,8 +163,11 @@ async def test_dense_recall_cosine_conversion( _make_row("t1", distance=0.2), _make_row("t2", distance=0.5), ] - tbl = _mock_ann_table(rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=10) assert len(result) == 2 @@ -203,12 +180,15 @@ async def test_dense_recall_cosine_conversion( async def test_dense_recall_empty_vector_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty vector short-circuits — no LanceDB query is issued.""" - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + """Empty vector short-circuits — no repo query is issued.""" + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + ) as mock_dense: result = await recaller.dense_recall([], _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_dense.assert_not_called() async def test_dense_recall_metadata_excludes_noise_columns( @@ -218,8 +198,11 @@ async def test_dense_recall_metadata_excludes_noise_columns( row = _make_row("t1", distance=0.3) row["vector"] = [0.0] * 1024 - tbl = _mock_ann_table([row]) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=[row], + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=5) assert len(result) == 1 diff --git a/tests/unit/test_memory/test_search/test_recall_or_semantics.py b/tests/unit/test_memory/test_search/test_recall_or_semantics.py index 3e35cde83..3c55a3a4b 100644 --- a/tests/unit/test_memory/test_search/test_recall_or_semantics.py +++ b/tests/unit/test_memory/test_search/test_recall_or_semantics.py @@ -24,13 +24,14 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.index.lancedb import build_or_query from everos.infra.persistence.lancedb import ( Episode, ParentType, episode_repo, lancedb_manager, ) -from everos.memory.search.recall.base import RecallerDeps, build_or_query +from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.episode import EpisodeRecaller @@ -92,16 +93,15 @@ def _recaller() -> EpisodeRecaller: def test_build_or_query_empty_returns_none() -> None: """Empty / whitespace-only query → ``None`` (caller must short-circuit).""" - tk = _WhitespaceTokenizer() - assert build_or_query(tk, "", column="episode_tokens") is None - assert build_or_query(tk, " ", column="episode_tokens") is None + assert build_or_query([], column="episode_tokens") is None + assert build_or_query(["", ""], column="episode_tokens") is None def test_build_or_query_single_token_returns_match_query() -> None: """One token → bare MatchQuery (no boolean-wrapper overhead).""" from lancedb.query import MatchQuery - q = build_or_query(_WhitespaceTokenizer(), "hello", column="episode_tokens") + q = build_or_query(["hello"], column="episode_tokens") assert isinstance(q, MatchQuery) @@ -109,9 +109,7 @@ def test_build_or_query_multi_token_returns_boolean_query() -> None: """≥2 tokens → BooleanQuery with one SHOULD clause per token.""" from lancedb.query import BooleanQuery - q = build_or_query( - _WhitespaceTokenizer(), "alice support group", column="episode_tokens" - ) + q = build_or_query(["alice", "support", "group"], column="episode_tokens") assert isinstance(q, BooleanQuery) @@ -148,7 +146,9 @@ async def test_or_semantics_poison_token_does_not_kill_query() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() - where = "owner_id = 'alice' AND owner_type = 'user'" + from everos.infra.persistence.index import all_of, eq + + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("alice support group", where, limit=10) assert cands, "alice + support + group should recall ep_1 via SHOULD" # ep_1 is the support-group episode; should rank above ep_2 (no support). @@ -177,7 +177,9 @@ async def test_or_semantics_single_informative_token() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() - where = "owner_id = 'alice' AND owner_type = 'user'" + from everos.infra.persistence.index import all_of, eq + + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("painting", where, limit=10) assert cands, "single informative token must recall the matching episode" assert cands[0].id == "alice_ep_2" @@ -185,5 +187,7 @@ async def test_or_semantics_single_informative_token() -> None: async def test_or_semantics_empty_query_returns_empty() -> None: """Tokenisation yields nothing → recall returns ``[]`` without hitting LanceDB.""" - cands = await _recaller().sparse_recall(" ", "owner_id = 'alice'", limit=10) + from everos.infra.persistence.index import eq + + cands = await _recaller().sparse_recall(" ", eq("owner_id", "alice"), limit=10) assert cands == [] diff --git a/tests/unit/test_service/test_knowledge_search.py b/tests/unit/test_service/test_knowledge_search.py index 73500924f..88bb7a0fb 100644 --- a/tests/unit/test_service/test_knowledge_search.py +++ b/tests/unit/test_service/test_knowledge_search.py @@ -135,12 +135,17 @@ def _patch_stack( class TestCompileKnowledgeWhere: def test_basic_clause(self) -> None: result = compile_knowledge_where("myapp", "myproj") - assert result == "app_id = 'myapp' AND project_id = 'myproj'" + from everos.infra.persistence.index import all_of, eq + + assert result == all_of(eq("app_id", "myapp"), eq("project_id", "myproj")) def test_defaults(self) -> None: result = compile_knowledge_where("default", "default") - assert "app_id = 'default'" in result - assert "project_id = 'default'" in result + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(result) + assert "app_id = 'default'" in rendered + assert "project_id = 'default'" in rendered def test_rejects_invalid_app_id_with_sql_injection(self) -> None: with pytest.raises(ValueError, match="app_id"): @@ -160,13 +165,19 @@ def test_rejects_empty_project_id(self) -> None: def test_accepts_valid_ids_with_special_chars(self) -> None: result = compile_knowledge_where("my_app.v2", "project-1") - assert "my_app.v2" in result - assert "project-1" in result + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(result) + assert "my_app.v2" in rendered + assert "project-1" in rendered def test_accepts_valid_ids_with_at_plus(self) -> None: result = compile_knowledge_where("app@org+v1", "proj_1") - assert "app@org+v1" in result - assert "proj_1" in result + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(result) + assert "app@org+v1" in rendered + assert "proj_1" in rendered # ── search_knowledge ───────────────────────────────────────────────────────── diff --git a/uv.lock b/uv.lock index ec09b1abb..5d1c15895 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,14 @@ version = 1 revision = 3 requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] [[package]] name = "aiosqlite" @@ -99,6 +107,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "cachetools" +version = "7.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/d2/47e8bc06fe2a06d3f5bdf20f1126ab66c4e99dc48d940e7ba873f7ac7131/cachetools-7.1.7.tar.gz", hash = "sha256:a3e2a00b14d8f8a6b70c1dae7b4685e7ad3bc965c5b42124a2d6ce895da6cf50", size = 40680, upload-time = "2026-08-01T21:20:40.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/d8/767faeda872075724b95dd675466a645f1b92aadcdcf2d1429dcfd76c176/cachetools-7.1.7-py3-none-any.whl", hash = "sha256:ef98ef375ad188819ef2f9b3645e3987f4b8c5b7550e436ad998c2de78296df0", size = 16830, upload-time = "2026-08-01T21:20:38.977Z" }, +] + [[package]] name = "cairocffi" version = "1.7.1" @@ -595,6 +612,9 @@ dependencies = [ ] [package.optional-dependencies] +milvus = [ + { name = "pymilvus" }, +] multimodal = [ { name = "everalgo-parser", extra = ["svg"] }, ] @@ -611,6 +631,7 @@ dev = [ { name = "opentelemetry-sdk" }, { name = "pre-commit" }, { name = "pyinstrument" }, + { name = "pymilvus" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -641,6 +662,7 @@ requires-dist = [ { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.7.1" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, + { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=3.0.0" }, { name = "python-multipart", specifier = ">=0.0.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlmodel", specifier = ">=0.0.22" }, @@ -651,7 +673,7 @@ requires-dist = [ { name = "watchdog", specifier = ">=4.0.0" }, { name = "watchfiles", specifier = ">=0.21.0" }, ] -provides-extras = ["multimodal", "otel"] +provides-extras = ["multimodal", "otel", "milvus"] [package.metadata.requires-dev] dev = [ @@ -661,6 +683,7 @@ dev = [ { name = "opentelemetry-sdk", specifier = ">=1.27.0" }, { name = "pre-commit", specifier = ">=4.0.0" }, { name = "pyinstrument", specifier = ">=5.0.0" }, + { name = "pymilvus", specifier = ">=3.0.0" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-asyncio", specifier = ">=1.1.0" }, { name = "pytest-cov", specifier = ">=6.0.0" }, @@ -850,6 +873,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/8f/774ce522de6a7e70fbeceeaeb6fbe502f5dfb8365728fb3bb4cb23463da8/grimp-3.14-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a424ad14d5deb56721ac24ab939747f72ab3d378d42e7d1f038317d33b052b77", size = 2515157, upload-time = "2025-12-10T17:54:55.874Z" }, ] +[[package]] +name = "grpcio" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" }, + { url = "https://files.pythonhosted.org/packages/39/33/b5b50fc2c6fbe350e04814047bb2d409feec7b36ef8b170254c050e06bc0/grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da", size = 12160538, upload-time = "2026-07-23T15:19:35.958Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" }, + { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" }, + { url = "https://files.pythonhosted.org/packages/17/5b/77af31228f55f55a2a5112bb0077ad0a1c4d23dbb0c2853a62475bbdcc14/grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc", size = 4394004, upload-time = "2026-07-23T15:19:50.618Z" }, + { url = "https://files.pythonhosted.org/packages/c0/da/f706e39550e7a3732ce2b9c5926107a93d74a802775b19b642a6df27dc96/grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df", size = 5158525, upload-time = "2026-07-23T15:19:52.246Z" }, + { url = "https://files.pythonhosted.org/packages/56/eb/135daaa713f32d33b8f99b4153b3f8dc3b2a124996ac15581bf9ebdad3c3/grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0", size = 6304480, upload-time = "2026-07-23T15:19:53.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a1/121806ce69f23138dabe06aa595b0e5f1ae051a37e4c1954eed7d692c800/grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1", size = 12154419, upload-time = "2026-07-23T15:19:56.3Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e8/d0389e09cd6b4c4d3089b92967ae4e3ffd64795bd349bf2f85cd6656d3da/grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867", size = 6873200, upload-time = "2026-07-23T15:19:58.701Z" }, + { url = "https://files.pythonhosted.org/packages/f8/51/f464c1d211fa50d5adbabe1b2e519948d99c13757052bfc9ea7afa28e284/grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881", size = 7618811, upload-time = "2026-07-23T15:20:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c0/539fe0832f2dd6500a28f5263071623fb34e8d4867aec632ccf81bd21156/grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f", size = 7042310, upload-time = "2026-07-23T15:20:02.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ca/ccf617d37ffa72567fa8e005ec7090c99da922799be2fb9847c8b21ca18c/grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61", size = 7575412, upload-time = "2026-07-23T15:20:04.712Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b9/fd8d5245f823a8e0fd35d90e20ea3aa4acd47f8d5318fa8df307df52dec6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45", size = 8604248, upload-time = "2026-07-23T15:20:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/14/1e/f37632fc11db72dfa4bba86c3a43e54358e53030df111ecae5e91a733ad6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c", size = 7977458, upload-time = "2026-07-23T15:20:09.109Z" }, + { url = "https://files.pythonhosted.org/packages/93/b6/d70b69ae5c0cfc341b9ba474980e4ed99cbf05c0e4a14e9eee8cb73db0a5/grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf", size = 4393993, upload-time = "2026-07-23T15:20:11.017Z" }, + { url = "https://files.pythonhosted.org/packages/0f/13/45d4cccb555cf4c476226979bf3d2fd0b0254216f7564c3a053e35117efc/grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735", size = 5159650, upload-time = "2026-07-23T15:20:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ab/d3874931d123a95e83a3ebf8aa04537988fb62425cedb8bf3cefc5ad41b2/grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c", size = 12166617, upload-time = "2026-07-23T15:20:17.435Z" }, + { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" }, + { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" }, + { url = "https://files.pythonhosted.org/packages/83/fa/c3fda157287f64bc65acee6c5aa90c41acf9e0d3a8e69a265eecff6d00a1/grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33", size = 4471765, upload-time = "2026-07-23T15:20:33.195Z" }, + { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1446,6 +1510,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] +[[package]] +name = "orjson" +version = "3.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/f3/742fb1f62b825f2c010697eaf4e828004bc2a81e7e806666989c132c7c42/orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5", size = 4142915, upload-time = "2026-08-14T16:13:30.607Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/4a/295da39c651c2faac8bd351a2a346f0fdedd9d50b847ee9dfc27d2207ef6/orjson-3.12.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:aa3e43a6846e91d7bde3d5a9c66090fcd8744f569a9b6cffc5e1ca38f6a461c0", size = 223427, upload-time = "2026-08-14T16:12:28.525Z" }, + { url = "https://files.pythonhosted.org/packages/29/98/758cf90fbeaaafb7f8141bfac75a432099959f3a2f5db93a412e876415d8/orjson-3.12.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:11edb4660a6680abee9788a3a9072208a2c96538cc1322bd79542065229d8e54", size = 123725, upload-time = "2026-08-14T16:12:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/32/b5/5b934d251f8651f7e41df180ad0c57a6e1cabe15c7bd331638413a50ebc9/orjson-3.12.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d3a9da945a4d96ae758fdaaca56742e6b73b6fd554c5d8876f252a6dad70b83", size = 113375, upload-time = "2026-08-14T16:12:31.209Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d2/37efb5b12a176ce3ced29f4144f20da57d02757f78ce549637dc1b4e1fc8/orjson-3.12.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:92ffc09e07233a6ab6d4e067f7841edcbcc134cb4812155cf171ea5255a421d7", size = 129983, upload-time = "2026-08-14T16:12:32.721Z" }, + { url = "https://files.pythonhosted.org/packages/50/22/0644b87c73f13e0092df8f35a1fe280d991e5e90072087411e0dd7e44e0c/orjson-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf44e374aadde77b1f6109f1030be51433eb61984379852766b6f4e187db7b1e", size = 130629, upload-time = "2026-08-14T16:12:34.084Z" }, + { url = "https://files.pythonhosted.org/packages/8c/57/80b986ebfecd9c6a177ddf1c2319717f0cd8feffb2b78946595a18a2fc88/orjson-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1192a7021b6d071aaf909864f6e924d6a2675ca360485b972b8401749311750b", size = 131245, upload-time = "2026-08-14T16:12:35.713Z" }, + { url = "https://files.pythonhosted.org/packages/80/3d/75c5ac5a69161f44492a68fbdde66f4cc4ce48cd5e1fb05918e46f0c8848/orjson-3.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:53c0c474a9d9aff9aebfc0c88de1f28f843d940e6e3a80729abdf6a20274356f", size = 135397, upload-time = "2026-08-14T16:12:37.128Z" }, + { url = "https://files.pythonhosted.org/packages/71/93/4d71f2df314a97ff0d27a4559bf5888fc8406e3c6dec90e92291e3511215/orjson-3.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:532ff8cd4bd59a327a953a7dcde922c7fc25b85e29721bb8633265430d3a3873", size = 127693, upload-time = "2026-08-14T16:12:38.627Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1d/0dbc6be5adfd1730491072fb60beb6bcdf5d7b2596ee41b7fc2e298bfc09/orjson-3.12.0-cp312-cp312-win32.whl", hash = "sha256:a6cf4b18e7de173f209f2084ffbd736dd72389a396326ee80a7022168be232e5", size = 128000, upload-time = "2026-08-14T16:12:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c9/97b1ce0112ebf5e949c775ed5b1755e562233179f3584579673cc24d6378/orjson-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:010811c1b69773450a01cef97727a67b223242f350b77d4ca000e59a9ef2155a", size = 122106, upload-time = "2026-08-14T16:12:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/a8/6a/facd8b312e4a0d3a7fa978c7e15821f74a336adf1d65529faec33b48e18b/orjson-3.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:ad29eece0c601737f2a60edc2752a84e7a0785df3efb62e3012834700a5afe0d", size = 126869, upload-time = "2026-08-14T16:12:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/54/cb/d7b78218a987eb8a8ce4eeae0286b1bb679333eb631ea0eeaf6371680bfc/orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900", size = 223397, upload-time = "2026-08-14T16:12:44.003Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4a/bc87c45e7ec639d35ebefd62618e01939531ac8e171426606a01bda05914/orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03", size = 123662, upload-time = "2026-08-14T16:12:45.433Z" }, + { url = "https://files.pythonhosted.org/packages/94/ee/c9a4ff3f2dbedbbe9e635d0fa72c8866adede09b6335ef9644f53752f0d8/orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8", size = 113374, upload-time = "2026-08-14T16:12:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/75/09/3f330a026a796c8b4c97a6f429652a5e912e7065039bf96ed25e42aa7b25/orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94", size = 130029, upload-time = "2026-08-14T16:12:48.06Z" }, + { url = "https://files.pythonhosted.org/packages/7d/40/094cc53126a3d22f76cdf83b6ea67338bed01d774037621a785aa8e6e5ea/orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806", size = 130528, upload-time = "2026-08-14T16:12:49.362Z" }, + { url = "https://files.pythonhosted.org/packages/bc/74/89bb236deb9565f99434b13052bb40ddfcce4adf3afbfa3132ee7e421468/orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df", size = 131075, upload-time = "2026-08-14T16:12:50.692Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ac/1176360d762c01b5bd34acd56fc098e936c491363d8b6b397ad4aa475547/orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978", size = 135321, upload-time = "2026-08-14T16:12:52.114Z" }, + { url = "https://files.pythonhosted.org/packages/7a/02/bbd881c8b9276d50b998de38b4e97de8ace1aac940b0ee545aedbf65ed00/orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222", size = 127472, upload-time = "2026-08-14T16:12:53.517Z" }, + { url = "https://files.pythonhosted.org/packages/8e/02/a0934d7503e6dcbedd6afac3e7f3f8597fd09389949ad94d0f7540e9dbca/orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1", size = 128000, upload-time = "2026-08-14T16:12:55.14Z" }, + { url = "https://files.pythonhosted.org/packages/52/87/69f98f8d40faff103a965a5fbb83f08241b01beaf92badb5413fbc9358cc/orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2", size = 121841, upload-time = "2026-08-14T16:12:56.507Z" }, + { url = "https://files.pythonhosted.org/packages/e6/07/b83046a4e3cadcc0987d0f160696107c4af706a619b56e4ad01940cadadf/orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e", size = 126765, upload-time = "2026-08-14T16:12:57.806Z" }, + { url = "https://files.pythonhosted.org/packages/12/9d/3931253e6f3148abf2cbe14830367042a4806b362ea520df2303db188fb9/orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d", size = 223391, upload-time = "2026-08-14T16:12:59.184Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0e/b4a4f1e305367245877b967a0bad70fcf001d77c54ac4339a120b66fdae4/orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647", size = 123659, upload-time = "2026-08-14T16:13:00.548Z" }, + { url = "https://files.pythonhosted.org/packages/96/f3/6782c6fa85e2702bc66be183c3b421486167dcf266ee4dc1403fe3824870/orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c", size = 113337, upload-time = "2026-08-14T16:13:02.009Z" }, + { url = "https://files.pythonhosted.org/packages/bf/79/b32ab64bacda9d0fa4942ef483bd03cabf0eaf2be819ca9fb7ff610c559d/orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc", size = 130112, upload-time = "2026-08-14T16:13:03.404Z" }, + { url = "https://files.pythonhosted.org/packages/ee/49/6e6142999ca01509219be5e5a9c338a3e5ea011f63e91ff473fbbf3734ed/orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1", size = 130520, upload-time = "2026-08-14T16:13:04.798Z" }, + { url = "https://files.pythonhosted.org/packages/49/d0/3745af0a4cc9867784f29722929cec4d10bd1c877cd754b01ba6d96eb21a/orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a", size = 131053, upload-time = "2026-08-14T16:13:06.14Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f4/6fe5a22fa478fffb190e65c338c84df5c311ef597b363150a17cc57063c0/orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e", size = 135321, upload-time = "2026-08-14T16:13:07.544Z" }, + { url = "https://files.pythonhosted.org/packages/ff/41/b1b0ec30289646a81a76e2dbaae2686b96fcccb7cb0323dc1dd78cbc7875/orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f", size = 127485, upload-time = "2026-08-14T16:13:08.88Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2b/277404bdcc21c93b112b963655b76443ebfe828f8a3ff1de7d90f8850eb3/orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92", size = 128048, upload-time = "2026-08-14T16:13:10.305Z" }, + { url = "https://files.pythonhosted.org/packages/41/2b/395b36fa2b4ce7af70b651d715e88f80d884b2c2b14a6b53e84d554fb5f0/orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed", size = 121858, upload-time = "2026-08-14T16:13:11.634Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a3/833e895ff452859eebe75093d26691fe9108f1a7a6a08435d7a5780ea652/orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7", size = 126749, upload-time = "2026-08-14T16:13:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/58/64/99c8947ece10c17176af9aae85c4948f1d109da77440ec14d87239efaf73/orjson-3.12.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:01efac2074fffb4cb1ea3fab7861e9d0f2a26913854a972f5ac760525dbdaf6e", size = 223398, upload-time = "2026-08-14T16:13:14.694Z" }, + { url = "https://files.pythonhosted.org/packages/3e/30/cf983fe09f2731420fda097a9f7ef4343f47fa216c228961ad8f6da44f3d/orjson-3.12.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:ed4ca42bd55955aa34deedcfdfd0e0c31abf51143aae158ae2bc3520b626e517", size = 123655, upload-time = "2026-08-14T16:13:16.221Z" }, + { url = "https://files.pythonhosted.org/packages/11/50/9cb8ae73fa4749dbbc20f617004213b5ff01c20aaeec34c3f31124f2c1d8/orjson-3.12.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:40f92192227505acca4e2533ce565f8e6b9535f7d0d09b0968452f18b7376b38", size = 130515, upload-time = "2026-08-14T16:13:17.601Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0a/adb6ce1a5b5fbf9cb1790f9961bb668a0dd5429aadaf6cee044724681795/orjson-3.12.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:33efefcf5d88eaf400b47e2eba02f91f319bb9951be61ca500b7d536d3f2079d", size = 113327, upload-time = "2026-08-14T16:13:18.927Z" }, + { url = "https://files.pythonhosted.org/packages/51/5c/d17f61581d8dbdde7048f87a330fa24915edec38db4d72b381fec14fbb56/orjson-3.12.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:8e386b0bc0ddd7cd2056f884b5a0af33592bd01ac66a7ca4b42a65a7e7774a13", size = 130105, upload-time = "2026-08-14T16:13:20.317Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b7/938befcf33bee4704a92ecec6a2731224c539d939bf9429fd39396d28931/orjson-3.12.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:58c58e1de0006ffb580368d6793c36c7b0b021db066479cf281bf5061e732328", size = 131049, upload-time = "2026-08-14T16:13:21.719Z" }, + { url = "https://files.pythonhosted.org/packages/b0/15/cfa2021d64d5aa8bb5c9f604ef375e00ec8b657651b5dd650b1b7ad13df1/orjson-3.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:08231552159be266a7269555bd9f7c016aee7d9ad6dab06eb58796c5ccb7101c", size = 135320, upload-time = "2026-08-14T16:13:23.415Z" }, + { url = "https://files.pythonhosted.org/packages/1a/50/3e75dfe357c1e8f9e287c7a5740260ef15bd23a5299eae8d0835dcad5375/orjson-3.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a15f9a891bce5f5cc5d210e3ad8614d4d1b489a56448c099d6d2a7168b2d954a", size = 127488, upload-time = "2026-08-14T16:13:24.791Z" }, + { url = "https://files.pythonhosted.org/packages/11/a6/79aed402eb3ab284dc5b4791a7ad62c5875127de01b8e3f04bd92d551298/orjson-3.12.0-cp315-cp315-win32.whl", hash = "sha256:03091c8a64db4be38746597ceea68f33c238e27acd9bfe99fb59420224ae7a55", size = 128048, upload-time = "2026-08-14T16:13:26.217Z" }, + { url = "https://files.pythonhosted.org/packages/64/f7/2723e264aab7248c1ed6ecaad8e5d0cb866c0cffde75442102ffa7491aba/orjson-3.12.0-cp315-cp315-win_amd64.whl", hash = "sha256:2b7bcefb9f40fa242fa6b06377232c048e655747790829609168c01162f60578", size = 121860, upload-time = "2026-08-14T16:13:27.577Z" }, + { url = "https://files.pythonhosted.org/packages/82/56/630c9113ec8996778f1f0304b364b091b9a9db5fef5fdc17cca622f5ea24/orjson-3.12.0-cp315-cp315-win_arm64.whl", hash = "sha256:859fc4196855890150bb08e649b30d2c93b249b3e3edd0d3bb2231abf8aa8adc", size = 126754, upload-time = "2026-08-14T16:13:28.962Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -1455,6 +1571,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + [[package]] name = "parso" version = "0.8.7" @@ -1469,7 +1631,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -1878,6 +2040,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/97/03635143a12a5d941f545548b00f8ac39d35565321a2effb4154ed267338/pyinstrument-5.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:b6a71f5e7f53c86c9b476b30cf19509463a63581ef17ddbd8680fee37ae509db", size = 128164, upload-time = "2026-01-04T18:38:32.281Z" }, ] +[[package]] +name = "pymilvus" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "grpcio" }, + { name = "orjson" }, + { name = "pandas" }, + { name = "protobuf" }, + { name = "python-dotenv" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/78/6bd0dba340706bc63346af96f0ebe36ff17f75404f5de935fda94d476c98/pymilvus-3.0.1.tar.gz", hash = "sha256:c02389059088b18d6e598cd175541e445c772fab4926c5e527c4913be34887f1", size = 347593, upload-time = "2026-07-29T14:55:43.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/9d/7011887b29f452905745e8bd321f404068d5bfe78fe84e42c0b7cd81a065/pymilvus-3.0.1-py3-none-any.whl", hash = "sha256:c5a8d5c1fa1de7b416e3529d383d8cc2e7da2170433ffa4a2d9087e14f70171a", size = 386820, upload-time = "2026-07-29T14:55:44.279Z" }, +] + [[package]] name = "pytest" version = "9.0.3"