diff --git a/deploy/README.md b/deploy/README.md index 407fdefe7..9a7ca3a52 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1178,35 +1178,80 @@ failures and scope limits; functional production validation is not a maximum-cap ## Incremental-sync page index -Migration 019 adds `idx_servers_updated_page_cover` on `(server_name, version)` -including `updated_at`, `is_latest`, and `status`. Unlike the partial search -index, it includes deleted versions required by incremental sync. Incremental -queries select a page of keys before fetching payloads. They retain prepared -statement reuse, binary payload transfer, and PostgreSQL automatic plan selection. -Forcing unnamed text-format execution regressed concurrent throughput in staging. The -watermark comparison, cursor ordering, filters, and response format are unchanged. - -On a busy existing database, prebuild before deploying the application: +Migration 020 replaces migration 019's included-column timestamp index with +`idx_servers_updated_name_cover` on `(server_name, version, updated_at)` including +`is_latest` and `status`. It also replaces the single-column timestamp index with +`idx_servers_updated_time_cover` on `(updated_at DESC, server_name, version)`. +Both include deleted versions needed by incremental-sync clients. + +For plain `updated_since` queries with `limit=100`, the query first probes up to +513 matching keys in timestamp order. With at most 512 matches it sorts that +complete set into name/version order; otherwise it fetches the requested page in +name order. Other incremental shapes without substring search retain the smaller +129-key probe and 128-match boundary. The larger probe avoids switching to a +costly name scan for moderately sparse pages. To avoid repeatedly paying for that +probe on large histories, each app samples one in 16 eligible plain 100-row +requests and retains up to 32 query-strategy hints for five seconds. A hint +records a timestamp/name range containing more than 512 matches. Requests +covered by that range can skip the probe; PostgreSQL verifies cursor ordering +behind a materialized guard so this decision does not defeat generic-plan reuse. +Hints are separated by deletion filtering and are not used in caller-owned +transactions or additional filter shapes. They contain no response data: writes +can make a hint suboptimal until it expires, but every request still applies the +complete predicate to current data. Sampled queries return an internal marker +row that is removed before pagination, serialization and returned-row metrics. +Both branches use the full predicate and one statement snapshot, with +payloads fetched only for the selected page. Timestamp order never becomes the +public pagination order. Plain incremental queries with page sizes 1 or 100 use +cached statements with the validated page size embedded in the query; cutoffs and +cursors remain parameters. The exact page bound lets PostgreSQL consider reusing +a generic plan instead of repeatedly planning the common shapes. Additional +filters and other page sizes retain per-execution planning. This is specific to +the adaptive query and replacement indexes, not a global plan-cache setting. +The probe bounds returned keys, not all index entries examined. Very large recent +histories, sparse cursors and different filter mixes still need measurement. + +On a busy existing database, **prebuild both indexes before deploying** the +application. These steps assume migration 019 is already applied; older +deployments must also prebuild indexes for their earlier pending migrations. +Startup migrations run inside a transaction and cannot use +`CONCURRENTLY`; without a prebuild they use regular index creation, which blocks +writes during the build. Use a maintenance window if prebuilding is not possible. +Run these as separate statements in autocommit, outside a transaction: ```sql -SET application_name = 'registry-updated-page-index-build'; +SET application_name = 'registry-updated-listing-index-build'; SET lock_timeout = '5s'; SET statement_timeout = '10min'; -CREATE INDEX CONCURRENTLY idx_servers_updated_page_cover - ON public.servers (server_name, version) INCLUDE (updated_at, is_latest, status); +CREATE INDEX CONCURRENTLY idx_servers_updated_name_cover + ON public.servers (server_name, version, updated_at) INCLUDE (is_latest, status); +CREATE INDEX CONCURRENTLY idx_servers_updated_time_cover + ON public.servers (updated_at DESC, server_name, version); + +SELECT c.relname, i.indisvalid, i.indisready, pg_get_indexdef(i.indexrelid) +FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid +WHERE c.relname IN ('idx_servers_updated_name_cover', 'idx_servers_updated_time_cover'); ``` -Run in autocommit, outside a transaction. Verify the definition and both -`indisvalid` and `indisready` before deploying. The migration creates the index -on fresh databases and rejects an existing invalid index. A concurrent build -uses CPU, I/O, and disk space; monitor it and repair any incomplete build before -retrying. - -Compare recent, old, and empty incremental-sync windows, deep cursors, deleted -records, and latest/version filters. Measure SQL execution and buffer access as -well as end-to-end latency, pool queues, CPU, and actual completed request rate. -Rolling back the application can leave this additive index in place; removal -is optional and should happen only after all candidate instances are gone. +Check both definitions and both readiness flags before deployment. Concurrent +builds consume CPU, I/O and temporary disk space; the old indexes remain until +migration 020 succeeds. If a build fails, inspect its state, drop only that new +incomplete index with `DROP INDEX CONCURRENTLY`, and retry in autocommit. Do not +use `IF NOT EXISTS` to assume a failed prebuild is usable. + +Migration 020 validates the replacements, then drops `idx_servers_updated_page_cover` +and `idx_servers_updated_at` in the same transaction. Its five-second lock timeout +bounds lock acquisition, not the duration of an index build. A failure rolls back +the migration and retains the old indexes; investigate before retrying. It does +not change table data, connection limits, HPA targets or replica counts. + +An application rollback remains schema-compatible with the replacement indexes, +but its planner choices and performance may differ. Leave the replacements in +place during an application rollback and measure behavior; restoring the exact +old physical layout requires a separate concurrent prebuild and reviewed cleanup. +Validate sparse/empty windows, early/deep cursors, `limit=1`, publishing, deleted +records and latest/version/search filters before production deployment. Compare +completed request rate, latency, CPU and pool queues at matched workloads. ### Packed production workers diff --git a/internal/database/migrations/020_replace_updated_listing_indexes.sql b/internal/database/migrations/020_replace_updated_listing_indexes.sql new file mode 100644 index 000000000..45b34ce2f --- /dev/null +++ b/internal/database/migrations/020_replace_updated_listing_indexes.sql @@ -0,0 +1,51 @@ +-- Prebuild both indexes CONCURRENTLY on busy databases before deployment. +-- Startup migrations are transactional; see deploy/README.md for the prebuild. +-- A new name avoids silently reusing 019's included-column timestamp layout. +SET LOCAL lock_timeout = '5s'; + +CREATE INDEX IF NOT EXISTS idx_servers_updated_name_cover + ON servers (server_name, version, updated_at) INCLUDE (is_latest, status); +CREATE INDEX IF NOT EXISTS idx_servers_updated_time_cover + ON servers (updated_at DESC, server_name, version); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_index + WHERE indexrelid = 'idx_servers_updated_name_cover'::regclass + AND indrelid = 'servers'::regclass + AND indisvalid AND indisready + AND indnkeyatts = 3 AND indnatts = 5 + AND indoption::text = '0 0 0' + AND indexprs IS NULL AND indpred IS NULL + AND (SELECT am.amname FROM pg_class c JOIN pg_am am ON am.oid=c.relam WHERE c.oid=indexrelid) = 'btree' + AND pg_get_indexdef(indexrelid, 1, true) = 'server_name' + AND pg_get_indexdef(indexrelid, 2, true) = 'version' + AND pg_get_indexdef(indexrelid, 3, true) = 'updated_at' + AND pg_get_indexdef(indexrelid, 4, true) = 'is_latest' + AND pg_get_indexdef(indexrelid, 5, true) = 'status' + ) THEN + RAISE EXCEPTION 'idx_servers_updated_name_cover must be valid, ready, and have the expected definition; repair the prebuild before retrying migration 020'; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_index + WHERE indexrelid = 'idx_servers_updated_time_cover'::regclass + AND indrelid = 'servers'::regclass + AND indisvalid AND indisready + AND indnkeyatts = 3 AND indnatts = 3 + AND indoption::text = '3 0 0' + AND indexprs IS NULL AND indpred IS NULL + AND (SELECT am.amname FROM pg_class c JOIN pg_am am ON am.oid=c.relam WHERE c.oid=indexrelid) = 'btree' + AND pg_get_indexdef(indexrelid, 1, true) = 'updated_at' + AND pg_get_indexdef(indexrelid, 2, true) = 'server_name' + AND pg_get_indexdef(indexrelid, 3, true) = 'version' + ) THEN + RAISE EXCEPTION 'idx_servers_updated_time_cover must be valid, ready, and have the expected definition; repair the prebuild before retrying migration 020'; + END IF; +END +$$; + +-- Both replacements are usable before the old indexes are removed. The +-- transaction rolls back on lock timeout, retaining the old indexes on failure. +DROP INDEX IF EXISTS idx_servers_updated_page_cover; +DROP INDEX IF EXISTS idx_servers_updated_at; diff --git a/internal/database/postgres.go b/internal/database/postgres.go index c148c10d3..3e3a25a3a 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -22,9 +22,10 @@ import ( // PostgreSQL is an implementation of the Database interface using PostgreSQL type PostgreSQL struct { - pool *pgxpool.Pool - poolMetrics metric.Registration - listingMetrics metric.Int64Counter + updatedPageHints updatedPageHints + pool *pgxpool.Pool + poolMetrics metric.Registration + listingMetrics metric.Int64Counter } // Executor is an interface for executing queries (satisfied by both pgx.Tx and pgxpool.Pool) @@ -220,28 +221,22 @@ func addCursorCondition(cursor string, argIndex int) (string, []any, int) { } // filteredPageQuery fetches payloads only after selecting a filtered page's keys. -func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limitArg int, prepared ...bool) string { +func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limitExpression string, prepared bool, strategy updatedPageStrategy) string { if filter == nil || (filter.SubstringName == nil && filter.UpdatedSince == nil) { return ordinary } - // Find the page using only its keys before fetching the JSON payloads. - // The covering index can reject nonmatching names without visiting the - // heap. The materialized candidate query plans for a larger key batch: - // a small LIMIT otherwise assumes rare matches occur early in name order - // and can favor scanning most of the ordered index over a selective GIN - // lookup. The outer LIMIT still defines the exact page before payload fetch. - // PostgreSQL 17 can stop the ordered CTE scan after the requested rows. + // Select the page using only its keys before fetching JSON payloads. payload := "s.value" - if len(prepared) > 0 && prepared[0] { + if prepared { payload = listingPayload("s.") } - // Incremental sync needs only the exact page. Its all-status covering index - // includes deleted versions, which consumers need to remove local records. - // Retain the larger planning boundary only for substring searches. - candidateLimit := fmt.Sprintf("$%d", limitArg) - if filter.SubstringName != nil { - candidateLimit = fmt.Sprintf("GREATEST($%d, 8192)", limitArg) + if filter.UpdatedSince != nil && filter.SubstringName == nil { + return updatedPageQuery(whereClause, limitExpression, payload, prepared, strategy) } + // For substring searches, a small LIMIT can favor scanning most of the + // ordered index over a selective GIN lookup. Plan for a larger key batch; + // the outer LIMIT still defines the exact page before payload fetch. + candidateLimit := fmt.Sprintf("GREATEST(%s, 8192)", limitExpression) return fmt.Sprintf(` WITH candidates AS MATERIALIZED ( SELECT server_name, version FROM servers @@ -254,18 +249,26 @@ func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limit FROM ( SELECT server_name, version FROM candidates ORDER BY server_name, version - LIMIT $%d + LIMIT %s ) page JOIN servers s USING (server_name, version) ORDER BY s.server_name, s.version - `, whereClause, candidateLimit, payload, limitArg) + `, whereClause, candidateLimit, payload, limitExpression) +} + +// cacheUpdatedPage bounds the extra statement-cache shapes to the two common +// page sizes. Other incremental filters retain parameter-specific planning. +func cacheUpdatedPage(filter *ServerFilter, limit int) bool { + return filter != nil && filter.UpdatedSince != nil && + filter.SubstringName == nil && filter.Name == nil && filter.Version == nil && + filter.IsLatest == nil && filter.RemoteURL == nil && (limit == 1 || limit == 100) } func filteredQueryArgs(filter *ServerFilter, args []any) []any { - if filter != nil && filter.SubstringName != nil { - // Search selectivity varies by orders of magnitude. Use an unnamed + if filter != nil && (filter.SubstringName != nil || filter.UpdatedSince != nil) { + // Search and incremental-sync selectivity vary greatly. Use an unnamed // extended-protocol query so PostgreSQL plans for these parameter values - // instead of reusing one generic plan for every term. + // instead of reusing one generic plan for every term, cutoff, and cursor. return append([]any{pgx.QueryExecModeExec}, args...) } return args @@ -334,6 +337,7 @@ func (db *PostgreSQL) listServerRows( whereConditions, args, argIndex := buildFilterConditions(filter, argIndex) // Add cursor pagination + cursorArgIndex := argIndex cursorCondition, cursorArgs, argIndex := addCursorCondition(cursor, argIndex) if cursorCondition != "" { whereConditions = append(whereConditions, cursorCondition) @@ -351,22 +355,35 @@ func (db *PostgreSQL) listServerRows( if prepared { payload = listingPayload("") } + cachePage := cacheUpdatedPage(filter, limit) + limitExpression := fmt.Sprintf("$%d", argIndex) + if cachePage { + // Only the validated constants 1 and 100 are embedded. Cutoffs and + // cursors remain parameters. This gives generic plans an exact page + // bound without forcing generic planning for other query shapes. + limitExpression = fmt.Sprintf("%d", limit) + } query := fmt.Sprintf(` SELECT server_name, version, status, status_changed_at, status_message, published_at, updated_at, is_latest, %s FROM servers %s ORDER BY server_name, version - LIMIT $%d - `, payload, whereClause, argIndex) - query = filteredPageQuery(filter, query, whereClause, argIndex, prepared) - args = append(args, limit) - args = filteredQueryArgs(filter, args) + LIMIT %s + `, payload, whereClause, limitExpression) + strategy, args := db.chooseUpdatedPageStrategy(tx, filter, limit, cursorArgs, cursorArgIndex, argIndex, args) + query = filteredPageQuery(filter, query, whereClause, limitExpression, prepared, strategy) + if cachePage { + args = append([]any{pgx.QueryExecModeCacheStatement}, args...) + } else { + args = append(args, limit) + args = filteredQueryArgs(filter, args) + } rows, err := db.getExecutor(tx).Query(ctx, query, args...) if err != nil { return nil, fmt.Errorf("failed to query servers: %w", err) } - return readListingRows(ctx, rows, diagnostic, prepared) + return db.readUpdatedPageRows(ctx, rows, diagnostic, prepared, strategy, filter) } type listingRow struct { @@ -376,7 +393,7 @@ type listingRow struct { prepared bool } -func readListingRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.ListingDiagnostic, prepared bool) ([]listingRow, error) { +func readListingRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.ListingDiagnostic, prepared bool, observation *updatedPageObservation) ([]listingRow, error) { defer rows.Close() // Copy the bounded page before decoding JSON. Exhausting/closing pooled rows @@ -386,7 +403,6 @@ func readListingRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.L var page []listingRow diagnostic.Phase("row_fetch") for rows.Next() { - diagnostic.RowReceived() diagnostic.Phase("row_fetch") if err := ctx.Err(); err != nil { return nil, err @@ -403,6 +419,10 @@ func readListingRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.L if err := rows.Scan(dest...); err != nil { return nil, fmt.Errorf("failed to scan server row: %w", err) } + if observation != nil && observation.consume(row) { + continue + } + diagnostic.RowReceived() meta.Status = model.Status(status) diagnostic.AddBytes(len(row.value)) page = append(page, row) diff --git a/internal/database/updated_indexes_internal_test.go b/internal/database/updated_indexes_internal_test.go new file mode 100644 index 000000000..b18fbaabe --- /dev/null +++ b/internal/database/updated_indexes_internal_test.go @@ -0,0 +1,80 @@ +package database + +import ( + "context" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +func TestUpdatedListingIndexMigration(t *testing.T) { + for _, mode := range []string{"fresh", "upgrade", "prebuilt", "wrong-definition"} { + t.Run(mode, func(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + conn, err := db.pool.Acquire(ctx) + require.NoError(t, err) + defer conn.Release() + migrator := NewMigrator(conn.Conn()) + migrations, err := migrator.loadMigrations() + require.NoError(t, err) + var migration Migration + for _, candidate := range migrations { + if candidate.Version == 20 { + migration = candidate + } + } + require.Equal(t, 20, migration.Version) + if mode != "fresh" { + _, err = conn.Exec(ctx, ` + DROP INDEX idx_servers_updated_name_cover; + DROP INDEX idx_servers_updated_time_cover; + CREATE INDEX idx_servers_updated_page_cover ON servers(server_name,version) INCLUDE(updated_at,is_latest,status); + CREATE INDEX idx_servers_updated_at ON servers(updated_at DESC); + DELETE FROM schema_migrations WHERE version=20; + `) + require.NoError(t, err) + switch mode { + case "prebuilt": + // Concurrent builds must be separate autocommit statements. + _, err = conn.Exec(ctx, `CREATE INDEX CONCURRENTLY idx_servers_updated_name_cover ON servers(server_name,version,updated_at) INCLUDE(is_latest,status)`) + require.NoError(t, err) + _, err = conn.Exec(ctx, `CREATE INDEX CONCURRENTLY idx_servers_updated_time_cover ON servers(updated_at DESC,server_name,version)`) + require.NoError(t, err) + case "wrong-definition": + // The old INCLUDE layout must not silently satisfy the new index name. + _, err = conn.Exec(ctx, `CREATE INDEX idx_servers_updated_name_cover ON servers(server_name,version) INCLUDE(updated_at,is_latest,status)`) + require.NoError(t, err) + } + err = migrator.applyMigration(ctx, migration) + if mode == "wrong-definition" { + require.ErrorContains(t, err, "expected definition") + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_page_cover", true) + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_at", true) + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_time_cover", false) + var applied bool + require.NoError(t, conn.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version=20)").Scan(&applied)) + require.False(t, applied) + return + } + require.NoError(t, err) + } + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_page_cover", false) + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_at", false) + for _, name := range []string{"idx_servers_updated_name_cover", "idx_servers_updated_time_cover"} { + var usable bool + require.NoError(t, conn.QueryRow(ctx, `SELECT indisvalid AND indisready AND indnkeyatts=3 FROM pg_index WHERE indexrelid=to_regclass($1)`, name).Scan(&usable)) + require.True(t, usable) + } + require.NoError(t, migrator.Migrate(ctx)) + }) + } +} + +func assertUpdatedIndexExists(t *testing.T, conn *pgx.Conn, name string, expected bool) { + t.Helper() + var exists bool + require.NoError(t, conn.QueryRow(context.Background(), "SELECT to_regclass($1) IS NOT NULL", name).Scan(&exists)) + require.Equal(t, expected, exists, name) +} diff --git a/internal/database/updated_page.go b/internal/database/updated_page.go new file mode 100644 index 000000000..096ca716b --- /dev/null +++ b/internal/database/updated_page.go @@ -0,0 +1,221 @@ +package database + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/modelcontextprotocol/registry/internal/telemetry" +) + +const ( + updatedPageProbe = 512 + updatedPageHintLimit = 32 + updatedPageHintLifetime = 5 * time.Second + updatedPageObservationInterval = 16 +) + +// updatedPageHints remembers query strategies, never response rows. A hint says +// a recent probe found >512 matching rows with minimum timestamp/name boundaries. +// Writes can invalidate that observation immediately: a stale hint may choose a +// slower name-ordered scan, but that scan still applies the complete predicate. +// Expiry bounds this performance risk; the fixed capacity bounds per-app memory. +// The zero value is ready to use. A PostgreSQL instance must not be copied. +type updatedPageHints struct { + mu sync.Mutex + requests uint64 + entries []updatedPageHint +} + +type updatedPageHint struct { + expires time.Time + before time.Time + nameBefore string + includeDeleted bool +} + +type updatedPageStrategy struct { + observe bool + skipProbe bool + nameBefore string + cursorGuard string // SQL over bound parameters, never interpolated cursor data. +} + +func (h *updatedPageHints) choose(now, since time.Time, cursorName string, includeDeleted bool) updatedPageStrategy { + h.mu.Lock() + defer h.mu.Unlock() + h.requests++ + strategy := updatedPageStrategy{observe: h.requests%updatedPageObservationInterval == 1} + for _, hint := range h.entries { + if hint.includeDeleted != includeDeleted || !now.Before(hint.expires) || !since.Before(hint.before) { + continue + } + // This screen can miss opportunities under a different DB collation. + // SQL rechecks strict name ordering before skipping a cursor-page probe. + if cursorName != "" && cursorName >= hint.nameBefore { + continue + } + if !strategy.skipProbe || hint.nameBefore > strategy.nameBefore { + strategy.skipProbe = true + strategy.nameBefore = hint.nameBefore + } + } + return strategy +} + +func (h *updatedPageHints) remember(now time.Time, observation updatedPageObservation, includeDeleted bool) { + h.mu.Lock() + defer h.mu.Unlock() + // Do not refresh an existing hint's lifetime without a fresh probe. A + // covered sampled query has no probe rows and does not produce a hint. + live := h.entries[:0] + for _, hint := range h.entries { + if now.Before(hint.expires) { + live = append(live, hint) + } + } + h.entries = live + for _, hint := range h.entries { + if hint.includeDeleted == includeDeleted && hint.before.Equal(observation.before) && hint.nameBefore == observation.nameBefore { + return + } + } + entry := updatedPageHint{expires: now.Add(updatedPageHintLifetime), before: observation.before, nameBefore: observation.nameBefore, includeDeleted: includeDeleted} + if len(h.entries) < updatedPageHintLimit { + h.entries = append(h.entries, entry) + return + } + oldest := 0 + for i := range h.entries { + if h.entries[i].expires.Before(h.entries[oldest].expires) { + oldest = i + } + } + h.entries[oldest] = entry +} + +// updatedPageObservation is returned in one private marker row on sampled +// queries. Empty server names are forbidden by the schema, so the marker cannot +// collide with a stored row. Reusing the row layout avoids widening every page +// row; these fields never reach API serialization or returned-row telemetry. +type updatedPageObservation struct { + large bool + before time.Time + nameBefore string +} + +func (o *updatedPageObservation) consume(row listingRow) bool { + if row.name != "" { + return false + } + o.large = row.metadata.IsLatest + o.before = row.metadata.UpdatedAt + if row.metadata.StatusMessage != nil { + o.nameBefore = *row.metadata.StatusMessage + } + return true +} + +// chooseUpdatedPageStrategy is limited to the common plain 100-row shape. +// Transaction-owned reads and additional filters never consult or populate hints. +func (db *PostgreSQL) chooseUpdatedPageStrategy(tx pgx.Tx, filter *ServerFilter, limit int, cursorArgs []any, cursorArgIndex, nextArg int, args []any) (updatedPageStrategy, []any) { + if tx != nil || limit != 100 || !cacheUpdatedPage(filter, limit) { + return updatedPageStrategy{}, args + } + includeDeleted := filter.IncludeDeleted != nil && *filter.IncludeDeleted + cursorName := "" + if len(cursorArgs) > 0 { + cursorName = cursorArgs[0].(string) + } + strategy := db.updatedPageHints.choose(time.Now(), *filter.UpdatedSince, cursorName, includeDeleted) + if strategy.skipProbe && len(cursorArgs) > 0 { + // PostgreSQL verifies ordering with server_name's default collation. + strategy.cursorGuard = fmt.Sprintf("($%d::text < $%d::text)", cursorArgIndex, nextArg) + args = append(args, strategy.nameBefore) + } + return strategy, args +} + +func (db *PostgreSQL) readUpdatedPageRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.ListingDiagnostic, prepared bool, strategy updatedPageStrategy, filter *ServerFilter) ([]listingRow, error) { + var observation *updatedPageObservation + if strategy.observe { + observation = &updatedPageObservation{} + } + page, err := readListingRows(ctx, rows, diagnostic, prepared, observation) + if err == nil && observation != nil && observation.large { + db.updatedPageHints.remember(time.Now(), *observation, filter.IncludeDeleted != nil && *filter.IncludeDeleted) + } + return page, err +} + +// updatedPageQuery selects keys before fetching payloads. A complete bounded +// timestamp probe is cheap for sparse histories; a name-ordered scan avoids +// sorting a large history. Both branches use one statement snapshot and the full +// predicate. The probe bounds returned keys, not every index entry visited. +func updatedPageQuery(whereClause, limitExpression, payload string, prepared bool, strategy updatedPageStrategy) string { + probe := 128 + if limitExpression == "100" { + probe = updatedPageProbe + } + prefix, covered := "", "FALSE" + if strategy.skipProbe { + covered = "TRUE" + if strategy.cursorGuard != "" { + // Constant folding otherwise makes early custom plans artificially + // cheap and can prevent PostgreSQL from reusing a generic plan. + prefix = "hint_guard AS MATERIALIZED (SELECT " + strategy.cursorGuard + " AS covers_page), " + covered = "(SELECT covers_page FROM hint_guard)" + } + } + columns, observation := "", "" + probeWhere := whereClause + choice := fmt.Sprintf("count(*) <= %d", probe) + if strategy.observe || strategy.skipProbe { + probeWhere += " AND NOT (" + covered + ")" + choice += " AND NOT (" + covered + ")" + } + if strategy.observe { + columns = ", updated_at" + observation = fmt.Sprintf(", count(*) > %d AS observed_large", probe) + } + ctes := fmt.Sprintf(`WITH %srecent AS MATERIALIZED ( + SELECT server_name, version%s FROM servers + %s + ORDER BY updated_at DESC LIMIT %d + ), choice AS MATERIALIZED ( + SELECT %s AS small%s FROM recent + ), candidates AS MATERIALIZED ( + (SELECT server_name, version FROM recent + WHERE (SELECT small FROM choice) + ORDER BY server_name, version LIMIT %s) + UNION ALL + (SELECT server_name, version FROM servers + %s AND NOT (SELECT small FROM choice) + ORDER BY server_name, version LIMIT %s) + )`, prefix, columns, probeWhere, probe+1, choice, observation, limitExpression, whereClause, limitExpression) + page := fmt.Sprintf(`SELECT s.server_name, s.version, s.status, s.status_changed_at, s.status_message, + s.published_at, s.updated_at, s.is_latest, %s + FROM (SELECT server_name, version FROM candidates + ORDER BY server_name, version LIMIT %s) page + JOIN servers s USING (server_name, version)`, payload, limitExpression) + if !strategy.observe { + return ctes + " " + page + " ORDER BY s.server_name, s.version" + } + return ctes + ", response_rows AS (" + page + ") SELECT * FROM response_rows UNION ALL " + + updatedPageMarker(prepared) + " ORDER BY server_name, version" +} + +func updatedPageMarker(prepared bool) string { + payload := "NULL::jsonb" + if prepared { + payload = "NULL::bytea, false" + } + return `SELECT ''::text AS server_name, ''::text AS version, ''::text AS status, + 'epoch'::timestamptz AS status_changed_at, + (SELECT CASE WHEN observed_large THEN (SELECT min(server_name) FROM recent) ELSE ''::text END FROM choice) AS status_message, + 'epoch'::timestamptz AS published_at, + (SELECT CASE WHEN observed_large THEN (SELECT min(updated_at) FROM recent) ELSE 'epoch'::timestamptz END FROM choice) AS updated_at, + (SELECT observed_large FROM choice) AS is_latest, ` + payload +} diff --git a/internal/database/updated_page_cache_internal_test.go b/internal/database/updated_page_cache_internal_test.go new file mode 100644 index 000000000..953cf8da3 --- /dev/null +++ b/internal/database/updated_page_cache_internal_test.go @@ -0,0 +1,89 @@ +package database + +import ( + "context" + "fmt" + "testing" + "time" + + apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0" + "github.com/modelcontextprotocol/registry/pkg/model" + "github.com/stretchr/testify/require" +) + +func TestUpdatedPageCachePreservesChangingWatermarksAndBounds(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + epoch := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + for i := 1; i <= 540; i++ { + stamp := epoch.Add(time.Duration(i) * time.Second) + _, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/cached-%03d", i), Version: "1.0.0", Description: "cached page fixture"}, &apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: epoch, UpdatedAt: stamp, StatusChangedAt: stamp, IsLatest: true}) + require.NoError(t, err) + } + // Keep one connection so each query shape crosses the prepared-plan threshold. + conn, err := db.pool.Acquire(ctx) + require.NoError(t, err) + defer conn.Release() + tx, err := conn.Begin(ctx) + require.NoError(t, err) + defer func() { require.NoError(t, tx.Rollback(ctx)) }() + yes := true + for round := 0; round < 3; round++ { + for _, count := range []int{1, 128, 129, 511, 512, 513, 540, 0, 21} { + cutoff := epoch.Add(time.Duration(540-count) * time.Second) + filter := &ServerFilter{UpdatedSince: &cutoff, IncludeDeleted: &yes} + for _, limit := range []int{1, 100} { + for _, after := range []int{0, 51} { + cursor := "" + if after > 0 { + cursor = fmt.Sprintf("com.example/cached-%03d:1.0.0", after) + } + rows, _, err := db.ListServers(ctx, tx, filter, cursor, limit) + require.NoError(t, err) + var want []string + for i := 1; i <= 540 && len(want) < limit; i++ { + if i > 540-count && i > after { + want = append(want, fmt.Sprintf("com.example/cached-%03d", i)) + } + } + var got []string + for _, row := range rows { + got = append(got, row.Server.Name) + } + require.Equal(t, want, got, "round=%d count=%d limit=%d after=%d", round, count, limit, after) + } + } + } + } + var cached int + require.NoError(t, tx.QueryRow(ctx, "SELECT count(*) FROM pg_prepared_statements WHERE statement ~ '^[[:space:]]*WITH recent AS MATERIALIZED'").Scan(&cached)) + require.Equal(t, 4, cached, "two page sizes, with and without a compound cursor") + var parameterizedLimit int + require.NoError(t, tx.QueryRow(ctx, "SELECT count(*) FROM pg_prepared_statements WHERE statement ~ '^[[:space:]]*WITH recent AS MATERIALIZED' AND statement LIKE '%LIMIT $%'").Scan(¶meterizedLimit)) + require.Zero(t, parameterizedLimit) + + // Changes are visible without reconnecting or clearing the cached statements. + _, err = tx.Exec(ctx, "UPDATE servers SET updated_at=$1 WHERE server_name LIKE 'com.example/cached-%'", epoch) + require.NoError(t, err) + rows, _, err := db.ListServers(ctx, tx, &ServerFilter{UpdatedSince: &epoch, IncludeDeleted: &yes}, "", 100) + require.NoError(t, err) + require.Empty(t, rows) + + // Keep additional filters and uncommon sizes on their existing custom path. + version, name, search, remote := "1.0.0", "com.example/cached-001", "cached", "https://example.invalid/mcp" + for _, filter := range []*ServerFilter{ + {UpdatedSince: &epoch, Version: &version}, {UpdatedSince: &epoch, Name: &name}, + {UpdatedSince: &epoch, SubstringName: &search}, {UpdatedSince: &epoch, IsLatest: &yes}, + {UpdatedSince: &epoch, RemoteURL: &remote}, + } { + _, _, err = db.ListServers(ctx, tx, filter, "", 100) + require.NoError(t, err) + } + for _, limit := range []int{0, 10, 128, 1000} { + _, _, err = db.ListServers(ctx, tx, &ServerFilter{UpdatedSince: &epoch}, "", limit) + require.NoError(t, err) + } + var after int + require.NoError(t, tx.QueryRow(ctx, "SELECT count(*) FROM pg_prepared_statements WHERE statement ~ '^[[:space:]]*WITH recent AS MATERIALIZED'").Scan(&after)) + require.Equal(t, cached, after) +} diff --git a/internal/database/updated_page_hints_internal_test.go b/internal/database/updated_page_hints_internal_test.go new file mode 100644 index 000000000..be7c070ba --- /dev/null +++ b/internal/database/updated_page_hints_internal_test.go @@ -0,0 +1,210 @@ +package database + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" + + "github.com/modelcontextprotocol/registry/internal/telemetry" + apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0" + "github.com/modelcontextprotocol/registry/pkg/model" +) + +func TestUpdatedPageHintsNeverCacheRows(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + epoch := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + yes := true + for i := 0; i < 540; i++ { + _, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/hint-%03d", i), Version: "1.0.0", Description: "hint fixture"}, &apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: epoch, UpdatedAt: epoch.Add(time.Second), StatusChangedAt: epoch, IsLatest: true}) + require.NoError(t, err) + } + // The private observation marker relies on this database invariant, even + // when a writer bypasses CreateServer's own name validation. + _, invalidNameErr := db.pool.Exec(ctx, "UPDATE servers SET server_name='' WHERE server_name='com.example/hint-000'") + require.ErrorContains(t, invalidNameErr, "check_server_name_format") + f := &ServerFilter{UpdatedSince: &epoch, IncludeDeleted: &yes} + read := func() ([]apiv0.ServerResponse, error) { + raw, err := db.ListServersJSON(ctx, f, "", 100) + if err != nil { + return nil, err + } + b, err := json.Marshal(raw) + if err != nil { + return nil, err + } + var out apiv0.ServerListResponse + err = json.Unmarshal(b, &out) + return out.Servers, err + } + page, err := read() + require.NoError(t, err) + require.Len(t, page, 100) + db.updatedPageHints.mu.Lock() + require.Len(t, db.updatedPageHints.entries, 1) + for k := range db.updatedPageHints.entries { + h := db.updatedPageHints.entries[k] + h.expires = time.Now().Add(time.Hour) + db.updatedPageHints.entries[k] = h + } + db.updatedPageHints.mu.Unlock() + _, err = db.pool.Exec(ctx, "UPDATE servers SET updated_at=$1", epoch) + require.NoError(t, err) + page, err = read() + require.NoError(t, err) + require.Empty(t, page) + _, err = db.pool.Exec(ctx, "UPDATE servers SET updated_at=$1 WHERE server_name < 'com.example/hint-030'", epoch.Add(time.Second)) + require.NoError(t, err) + page, err = read() + require.NoError(t, err) + require.Len(t, page, 30) + rows, _, err := db.ListServers(ctx, nil, f, "", 100) + require.NoError(t, err) + require.Len(t, rows, 30) + // Concurrent requests exercise the advisory map while all results stay current. + var wg sync.WaitGroup + errs := make(chan error, 16) + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 10; j++ { + r, e := read() + if e != nil { + errs <- e + return + } + if len(r) != 30 { + errs <- fmt.Errorf("rows=%d", len(r)) + return + } + } + }() + } + wg.Wait() + close(errs) + for e := range errs { + require.NoError(t, e) + } + db.updatedPageHints.mu.Lock() + for k := range db.updatedPageHints.entries { + h := db.updatedPageHints.entries[k] + h.expires = time.Now().Add(-time.Second) + db.updatedPageHints.entries[k] = h + } + db.updatedPageHints.mu.Unlock() + page, err = read() + require.NoError(t, err) + require.Len(t, page, 30) + _, err = db.pool.Exec(ctx, "UPDATE servers SET updated_at=$1", epoch.Add(time.Second)) + require.NoError(t, err) + page, err = read() + require.NoError(t, err) + require.Len(t, page, 100) +} + +// A moving sync watermark and cursor must not turn an algorithm hint into cached data. +func TestUpdatedPageHintsMovingWatermarkAndBoundedKeys(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + epoch := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + stamp := epoch.Add(time.Second) + yes := true + for i := 0; i < 540; i++ { + _, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/range-%03d", i), Version: "1.0.0", Description: "range fixture"}, &apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: epoch, UpdatedAt: stamp, StatusChangedAt: epoch, IsLatest: true}) + require.NoError(t, err) + } + read := func(since time.Time, cursor string, include bool) int { + f := &ServerFilter{UpdatedSince: &since, IncludeDeleted: &include} + raw, err := db.ListServersJSON(ctx, f, cursor, 100) + require.NoError(t, err) + b, err := json.Marshal(raw) + require.NoError(t, err) + var response apiv0.ServerListResponse + require.NoError(t, json.Unmarshal(b, &response)) + return len(response.Servers) + } + require.Equal(t, 100, read(epoch, "", yes)) + require.Equal(t, 100, read(epoch.Add(time.Millisecond), "", yes)) + require.Equal(t, 100, read(epoch, "com.example/aaa", yes), "legacy name-only cursor") + require.Equal(t, 39, read(epoch, "com.example/range-500", yes)) + require.Equal(t, 0, read(stamp, "", yes)) + require.Equal(t, 0, read(stamp.Add(time.Second), "", yes)) + require.Equal(t, 39, read(epoch, "com.example/range-500:1.0.0", yes)) + _, err := db.pool.Exec(ctx, "UPDATE servers SET updated_at=$1 WHERE server_name < 'com.example/range-040'", stamp.Add(time.Second)) + require.NoError(t, err) + require.Equal(t, 40, read(stamp, "", yes)) + _, err = db.pool.Exec(ctx, "UPDATE servers SET status='deleted' WHERE server_name < 'com.example/range-520'") + require.NoError(t, err) + require.Equal(t, 20, read(epoch, "", false)) + require.Equal(t, 100, read(epoch, "", yes)) + // Arbitrary valid cursor parameters cannot grow the advisory map without bound. + for i := 0; i < 540; i++ { + require.Equal(t, 100, read(epoch, fmt.Sprintf("com.example/aaa:%06d", i), yes)) + } + db.updatedPageHints.mu.Lock() + require.LessOrEqual(t, len(db.updatedPageHints.entries), 32) + for k, h := range db.updatedPageHints.entries { + h.expires = time.Now().Add(-time.Second) + db.updatedPageHints.entries[k] = h + } + db.updatedPageHints.mu.Unlock() + for i := 0; i < 16; i++ { + require.Equal(t, 100, read(epoch, "com.example/aab:1", yes)) + } + db.updatedPageHints.mu.Lock() + require.Len(t, db.updatedPageHints.entries, 1) + db.updatedPageHints.mu.Unlock() +} + +// Exercise the actual prepared guard shape on one connection: a custom-plan +// shortcut must not prevent generic reuse as watermarks/cursors change. +func TestUpdatedPageGuardReusesPlanAndChecksCursor(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + epoch := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i := 0; i < 540; i++ { + _, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/guard-%03d", i), Version: "1.0.0", Description: "guard fixture"}, &apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: epoch, UpdatedAt: epoch.Add(time.Second), StatusChangedAt: epoch, IsLatest: true}) + require.NoError(t, err) + } + conn, err := db.pool.Acquire(ctx) + require.NoError(t, err) + defer conn.Release() + strategy := updatedPageStrategy{skipProbe: true, cursorGuard: "($2::text < $4::text)"} + query := updatedPageQuery("WHERE updated_at > $1 AND (server_name, version) > ($2, $3)", "100", "s.value", false, strategy) + for i := 0; i < 24; i++ { + rows, err := conn.Query(ctx, query, pgx.QueryExecModeCacheStatement, epoch.Add(time.Duration(i)*time.Millisecond), "com.example/aaa", fmt.Sprint(i), "com.example/guard-000") + require.NoError(t, err) + page, err := readListingRows(ctx, rows, telemetry.ListingFromContext(ctx), false, nil) + require.NoError(t, err) + require.Len(t, page, 100) + require.Equal(t, "com.example/guard-000", page[0].name) + } + var generic, custom int + require.NoError(t, conn.QueryRow(ctx, "SELECT generic_plans, custom_plans FROM pg_prepared_statements WHERE statement = $1", query).Scan(&generic, &custom)) + require.Positive(t, generic, "guard must permit generic-plan reuse") + require.Less(t, custom, 24) + // Deliberately supply an unsuitable hint. The database guard rejects it; + // the timestamp probe still returns the sparse remaining page correctly. + rows, err := conn.Query(ctx, query, pgx.QueryExecModeCacheStatement, epoch, "com.example/guard-530", "1.0.0", "com.example/guard-000") + require.NoError(t, err) + page, err := readListingRows(ctx, rows, telemetry.ListingFromContext(ctx), false, nil) + require.NoError(t, err) + require.Len(t, page, 9) + require.Equal(t, "com.example/guard-531", page[0].name) + // An explicit transaction bypasses hints even when the cache is populated. + tx, err := conn.Begin(ctx) + require.NoError(t, err) + defer func() { require.NoError(t, tx.Rollback(ctx)) }() + db.updatedPageHints.remember(time.Now(), updatedPageObservation{large: true, before: epoch.Add(time.Second), nameBefore: "com.example/guard-000"}, false) + before := db.updatedPageHints.requests + _, _, err = db.ListServers(ctx, tx, &ServerFilter{UpdatedSince: &epoch}, "", 100) + require.NoError(t, err) + require.Equal(t, before, db.updatedPageHints.requests) +} diff --git a/internal/database/updated_page_strategy_internal_test.go b/internal/database/updated_page_strategy_internal_test.go new file mode 100644 index 000000000..144dc6d1c --- /dev/null +++ b/internal/database/updated_page_strategy_internal_test.go @@ -0,0 +1,115 @@ +package database + +import ( + "fmt" + "sync" + "testing" + "time" + + apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0" + "github.com/stretchr/testify/require" +) + +func TestUpdatedPageHintsCoverageAndExpiry(t *testing.T) { + var hints updatedPageHints + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + before := now.Add(-time.Hour) + hints.remember(now, updatedPageObservation{large: true, before: before, nameBefore: "com.example/m"}, false) + for _, tc := range []struct { + name string + at, since time.Time + cursor string + deleted, covered bool + }{ + {"first page", now, before.Add(-time.Second), "", false, true}, + {"earlier cursor", now, before.Add(-time.Second), "com.example/a", false, true}, + {"equal cursor", now, before.Add(-time.Second), "com.example/m", false, false}, + {"later cursor", now, before.Add(-time.Second), "com.example/z", false, false}, + {"equal watermark", now, before, "", false, false}, + {"later watermark", now, before.Add(time.Second), "", false, false}, + {"different deletion filter", now, before.Add(-time.Second), "", true, false}, + {"before expiry", now.Add(updatedPageHintLifetime - time.Nanosecond), before.Add(-time.Second), "", false, true}, + {"at expiry", now.Add(updatedPageHintLifetime), before.Add(-time.Second), "", false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got := hints.choose(tc.at, tc.since, tc.cursor, tc.deleted) + require.Equal(t, tc.covered, got.skipProbe) + }) + } + // Identical observations cannot indefinitely prolong an old strategy. + hints.remember(now.Add(time.Second), updatedPageObservation{large: true, before: before, nameBefore: "com.example/m"}, false) + require.Len(t, hints.entries, 1) + require.Equal(t, now.Add(updatedPageHintLifetime), hints.entries[0].expires) +} + +func TestUpdatedPageHintsBoundedSamplingAndConcurrentAccess(t *testing.T) { + var hints updatedPageHints + now := time.Now() + for i := 0; i < 64; i++ { + got := hints.choose(now, now.Add(-time.Hour), "", false) + require.Equal(t, i%updatedPageObservationInterval == 0, got.observe) + hints.remember(now.Add(time.Duration(i)*time.Millisecond), updatedPageObservation{ + large: true, before: now, nameBefore: fmt.Sprintf("com.example/%03d", i), + }, false) + } + require.Len(t, hints.entries, updatedPageHintLimit) + for _, hint := range hints.entries { + require.GreaterOrEqual(t, hint.nameBefore, "com.example/032", "oldest entries are evicted") + } + var wg sync.WaitGroup + for worker := 0; worker < 16; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 100; i++ { + hints.choose(now, now.Add(-time.Hour), "com.example/000", false) + hints.remember(now, updatedPageObservation{large: true, before: now, nameBefore: fmt.Sprintf("com.example/concurrent-%d", i)}, false) + } + }() + } + wg.Wait() + require.Len(t, hints.entries, updatedPageHintLimit) + hints.remember(now.Add(time.Minute), updatedPageObservation{large: true, before: now, nameBefore: "com.example/new"}, true) + require.Len(t, hints.entries, 1, "expired entries are removed on observation") +} + +func TestUpdatedPageObservationMarker(t *testing.T) { + stamp := time.Now() + name := "com.example/first" + ordinary := listingRow{name: name, metadata: apiv0.RegistryExtensions{UpdatedAt: stamp, IsLatest: true}} + var observation updatedPageObservation + require.False(t, observation.consume(ordinary)) + require.Equal(t, updatedPageObservation{}, observation) + marker := listingRow{metadata: apiv0.RegistryExtensions{UpdatedAt: stamp, StatusMessage: &name, IsLatest: true}} + require.True(t, observation.consume(marker)) + require.Equal(t, updatedPageObservation{large: true, before: stamp, nameBefore: name}, observation) +} + +func TestUpdatedPageStrategyScopeAndBoundParameters(t *testing.T) { + db := &PostgreSQL{} + now := time.Now() + cutoff := now.Add(-time.Hour) + name, version, search, remote, yes := "com.example/a", "1", "example", "https://example.com", true + for _, filter := range []*ServerFilter{ + nil, {}, {UpdatedSince: &cutoff, Name: &name}, {UpdatedSince: &cutoff, Version: &version}, + {UpdatedSince: &cutoff, SubstringName: &search}, {UpdatedSince: &cutoff, IsLatest: &yes}, + {UpdatedSince: &cutoff, RemoteURL: &remote}, + } { + strategy, _ := db.chooseUpdatedPageStrategy(nil, filter, 100, nil, 2, 2, nil) + require.Equal(t, updatedPageStrategy{}, strategy) + } + for _, limit := range []int{1, 10, 128, 1000} { + strategy, _ := db.chooseUpdatedPageStrategy(nil, &ServerFilter{UpdatedSince: &cutoff}, limit, nil, 2, 2, nil) + require.Equal(t, updatedPageStrategy{}, strategy) + } + require.Zero(t, db.updatedPageHints.requests) + db.updatedPageHints.remember(now, updatedPageObservation{large: true, before: now, nameBefore: "com.example/z'--"}, false) + cursor := "com.example/a' OR true --" + strategy, args := db.chooseUpdatedPageStrategy(nil, &ServerFilter{UpdatedSince: &cutoff}, 100, []any{cursor}, 2, 3, []any{cutoff, cursor}) + require.True(t, strategy.skipProbe) + require.Equal(t, "($2::text < $3::text)", strategy.cursorGuard) + require.Equal(t, []any{cutoff, cursor, "com.example/z'--"}, args) + query := updatedPageQuery("WHERE updated_at > $1 AND server_name > $2", "100", "s.value", false, strategy) + require.NotContains(t, query, cursor) + require.NotContains(t, query, strategy.nameBefore) +} diff --git a/internal/database/updated_page_test.go b/internal/database/updated_page_test.go index bdc8af738..232fd9ed7 100644 --- a/internal/database/updated_page_test.go +++ b/internal/database/updated_page_test.go @@ -103,6 +103,76 @@ func TestUpdatedPagePreservesWatermarkPaginationAndPayloads(t *testing.T) { } } +// Exercise both selection branches, including exact probe-boundary counts, +// using an independently constructed expected sequence and full pagination. +func TestUpdatedPageProbeBoundary(t *testing.T) { + db := database.NewTestDB(t) + ctx := context.Background() + epoch := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + yes, no := true, false + version := "1.0.0" + var fixture []*apiv0.ServerResponse + for i := 0; i < 260; i++ { + status := model.StatusActive + if i%3 == 0 { + status = model.StatusDeleted + } + stamp := epoch.Add(time.Duration(i+1) * time.Second) + r, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/boundary-%03d", i), Version: version, Description: "boundary payload"}, &apiv0.RegistryExtensions{Status: status, StatusChangedAt: stamp, PublishedAt: epoch, UpdatedAt: stamp, IsLatest: true}) + require.NoError(t, err) + fixture = append(fixture, r) + } + for _, count := range []int{0, 1, 127, 128, 129, 130, 260} { + cutoff := epoch.Add(time.Duration(260-count) * time.Second) + for _, includeDeleted := range []*bool{&yes, &no, nil} { + f := &database.ServerFilter{UpdatedSince: &cutoff, IncludeDeleted: includeDeleted, Version: &version} + want := expectedUpdatedServers(fixture, f, cutoff) + for _, limit := range []int{1, 100, 128, 129, 1000} { + var got []*apiv0.ServerResponse + cursor := "" + for page := 0; page < 262; page++ { + rows, next, err := db.ListServers(ctx, nil, f, cursor, limit) + require.NoError(t, err) + for _, r := range rows { + r.Meta.Official.PublishedAt = r.Meta.Official.PublishedAt.UTC() + r.Meta.Official.UpdatedAt = r.Meta.Official.UpdatedAt.UTC() + r.Meta.Official.StatusChangedAt = r.Meta.Official.StatusChangedAt.UTC() + } + wire, err := db.(*database.PostgreSQL).ListServersJSON(ctx, f, cursor, limit) + require.NoError(t, err) + encoded, err := json.Marshal(wire) + require.NoError(t, err) + var decoded apiv0.ServerListResponse + require.NoError(t, json.Unmarshal(encoded, &decoded)) + require.Equal(t, next, decoded.Metadata.NextCursor) + require.Len(t, decoded.Servers, len(rows)) + for i := range rows { + // Typed pgx timestamps can use the host timezone; compare + // the same instants regardless of the developer's locale. + meta := decoded.Servers[i].Meta.Official + meta.PublishedAt = meta.PublishedAt.UTC() + meta.UpdatedAt = meta.UpdatedAt.UTC() + meta.StatusChangedAt = meta.StatusChangedAt.UTC() + expectedJSON, err := json.Marshal(rows[i]) + require.NoError(t, err) + actualJSON, err := json.Marshal(decoded.Servers[i]) + require.NoError(t, err) + require.JSONEq(t, string(expectedJSON), string(actualJSON)) + } + got = append(got, rows...) + if next == "" { + break + } + require.NotEqual(t, cursor, next) + cursor = next + require.Less(t, page, 261) + } + require.Equal(t, want, got, "count=%d limit=%d", count, limit) + } + } + } +} + func expectedUpdatedServers(fixture []*apiv0.ServerResponse, filter *database.ServerFilter, cutoff time.Time) []*apiv0.ServerResponse { var want []*apiv0.ServerResponse for _, server := range fixture {