From adddab2f9619088c345bbf758bc887fa34723941 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Wed, 18 Nov 2026 11:17:05 +0200 Subject: [PATCH 1/4] Adapt updated_since page selection and replace listing indexes --- deploy/README.md | 75 +++++++++++------ .../020_replace_updated_listing_indexes.sql | 51 ++++++++++++ internal/database/postgres.go | 58 ++++++++++---- .../database/updated_indexes_internal_test.go | 80 +++++++++++++++++++ internal/database/updated_page_test.go | 64 +++++++++++++++ 5 files changed, 288 insertions(+), 40 deletions(-) create mode 100644 internal/database/migrations/020_replace_updated_listing_indexes.sql create mode 100644 internal/database/updated_indexes_internal_test.go diff --git a/deploy/README.md b/deploy/README.md index 407fdefe7..52232ac36 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1178,35 +1178,64 @@ failures and scope limits; functional production validation is not a maximum-cap ## Incremental-sync page index -Migration 019 adds `idx_servers_updated_page_cover` on `(server_name, version)` -including `updated_at`, `is_latest`, and `status`. Unlike the partial search -index, it includes deleted versions required by incremental sync. Incremental -queries select a page of keys before fetching payloads. They retain prepared -statement reuse, binary payload transfer, and PostgreSQL automatic plan selection. -Forcing unnamed text-format execution regressed concurrent throughput in staging. The -watermark comparison, cursor ordering, filters, and response format are unchanged. - -On a busy existing database, prebuild before deploying the application: +Migration 020 replaces migration 019's included-column timestamp index with +`idx_servers_updated_name_cover` on `(server_name, version, updated_at)` including +`is_latest` and `status`. It also replaces the single-column timestamp index with +`idx_servers_updated_time_cover` on `(updated_at DESC, server_name, version)`. +Both include deleted versions needed by incremental-sync clients. + +For `updated_since` without substring search, the query first probes up to 129 +matching keys in timestamp order. With at most 128 matches it sorts that complete +set into name/version order; otherwise it fetches the requested page in name +order. Both branches use the full predicate and one statement snapshot, with +payloads fetched only for the selected page. Timestamp order never becomes the +public pagination order. Incremental queries use per-execution planning because +cutoff and cursor selectivity vary; this must be evaluated with the query and +index combination, not as a general recommendation to disable prepared plans. +The probe bounds returned keys, not all index entries examined. Very large recent +histories, sparse cursors and different filter mixes still need measurement. + +On a busy existing database, **prebuild both indexes before deploying** the +application. These steps assume migration 019 is already applied; older +deployments must also prebuild indexes for their earlier pending migrations. +Startup migrations run inside a transaction and cannot use +`CONCURRENTLY`; without a prebuild they use regular index creation, which blocks +writes during the build. Use a maintenance window if prebuilding is not possible. +Run these as separate statements in autocommit, outside a transaction: ```sql -SET application_name = 'registry-updated-page-index-build'; +SET application_name = 'registry-updated-listing-index-build'; SET lock_timeout = '5s'; SET statement_timeout = '10min'; -CREATE INDEX CONCURRENTLY idx_servers_updated_page_cover - ON public.servers (server_name, version) INCLUDE (updated_at, is_latest, status); +CREATE INDEX CONCURRENTLY idx_servers_updated_name_cover + ON public.servers (server_name, version, updated_at) INCLUDE (is_latest, status); +CREATE INDEX CONCURRENTLY idx_servers_updated_time_cover + ON public.servers (updated_at DESC, server_name, version); + +SELECT c.relname, i.indisvalid, i.indisready, pg_get_indexdef(i.indexrelid) +FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid +WHERE c.relname IN ('idx_servers_updated_name_cover', 'idx_servers_updated_time_cover'); ``` -Run in autocommit, outside a transaction. Verify the definition and both -`indisvalid` and `indisready` before deploying. The migration creates the index -on fresh databases and rejects an existing invalid index. A concurrent build -uses CPU, I/O, and disk space; monitor it and repair any incomplete build before -retrying. - -Compare recent, old, and empty incremental-sync windows, deep cursors, deleted -records, and latest/version filters. Measure SQL execution and buffer access as -well as end-to-end latency, pool queues, CPU, and actual completed request rate. -Rolling back the application can leave this additive index in place; removal -is optional and should happen only after all candidate instances are gone. +Check both definitions and both readiness flags before deployment. Concurrent +builds consume CPU, I/O and temporary disk space; the old indexes remain until +migration 020 succeeds. If a build fails, inspect its state, drop only that new +incomplete index with `DROP INDEX CONCURRENTLY`, and retry in autocommit. Do not +use `IF NOT EXISTS` to assume a failed prebuild is usable. + +Migration 020 validates the replacements, then drops `idx_servers_updated_page_cover` +and `idx_servers_updated_at` in the same transaction. Its five-second lock timeout +bounds lock acquisition, not the duration of an index build. A failure rolls back +the migration and retains the old indexes; investigate before retrying. It does +not change table data, connection limits, HPA targets or replica counts. + +An application rollback remains schema-compatible with the replacement indexes, +but its planner choices and performance may differ. Leave the replacements in +place during an application rollback and measure behavior; restoring the exact +old physical layout requires a separate concurrent prebuild and reviewed cleanup. +Validate sparse/empty windows, early/deep cursors, `limit=1`, publishing, deleted +records and latest/version/search filters before production deployment. Compare +completed request rate, latency, CPU and pool queues at matched workloads. ### Packed production workers diff --git a/internal/database/migrations/020_replace_updated_listing_indexes.sql b/internal/database/migrations/020_replace_updated_listing_indexes.sql new file mode 100644 index 000000000..45b34ce2f --- /dev/null +++ b/internal/database/migrations/020_replace_updated_listing_indexes.sql @@ -0,0 +1,51 @@ +-- Prebuild both indexes CONCURRENTLY on busy databases before deployment. +-- Startup migrations are transactional; see deploy/README.md for the prebuild. +-- A new name avoids silently reusing 019's included-column timestamp layout. +SET LOCAL lock_timeout = '5s'; + +CREATE INDEX IF NOT EXISTS idx_servers_updated_name_cover + ON servers (server_name, version, updated_at) INCLUDE (is_latest, status); +CREATE INDEX IF NOT EXISTS idx_servers_updated_time_cover + ON servers (updated_at DESC, server_name, version); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_index + WHERE indexrelid = 'idx_servers_updated_name_cover'::regclass + AND indrelid = 'servers'::regclass + AND indisvalid AND indisready + AND indnkeyatts = 3 AND indnatts = 5 + AND indoption::text = '0 0 0' + AND indexprs IS NULL AND indpred IS NULL + AND (SELECT am.amname FROM pg_class c JOIN pg_am am ON am.oid=c.relam WHERE c.oid=indexrelid) = 'btree' + AND pg_get_indexdef(indexrelid, 1, true) = 'server_name' + AND pg_get_indexdef(indexrelid, 2, true) = 'version' + AND pg_get_indexdef(indexrelid, 3, true) = 'updated_at' + AND pg_get_indexdef(indexrelid, 4, true) = 'is_latest' + AND pg_get_indexdef(indexrelid, 5, true) = 'status' + ) THEN + RAISE EXCEPTION 'idx_servers_updated_name_cover must be valid, ready, and have the expected definition; repair the prebuild before retrying migration 020'; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_index + WHERE indexrelid = 'idx_servers_updated_time_cover'::regclass + AND indrelid = 'servers'::regclass + AND indisvalid AND indisready + AND indnkeyatts = 3 AND indnatts = 3 + AND indoption::text = '3 0 0' + AND indexprs IS NULL AND indpred IS NULL + AND (SELECT am.amname FROM pg_class c JOIN pg_am am ON am.oid=c.relam WHERE c.oid=indexrelid) = 'btree' + AND pg_get_indexdef(indexrelid, 1, true) = 'updated_at' + AND pg_get_indexdef(indexrelid, 2, true) = 'server_name' + AND pg_get_indexdef(indexrelid, 3, true) = 'version' + ) THEN + RAISE EXCEPTION 'idx_servers_updated_time_cover must be valid, ready, and have the expected definition; repair the prebuild before retrying migration 020'; + END IF; +END +$$; + +-- Both replacements are usable before the old indexes are removed. The +-- transaction rolls back on lock timeout, retaining the old indexes on failure. +DROP INDEX IF EXISTS idx_servers_updated_page_cover; +DROP INDEX IF EXISTS idx_servers_updated_at; diff --git a/internal/database/postgres.go b/internal/database/postgres.go index c148c10d3..3bef0be24 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -224,24 +224,48 @@ func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limit if filter == nil || (filter.SubstringName == nil && filter.UpdatedSince == nil) { return ordinary } - // Find the page using only its keys before fetching the JSON payloads. - // The covering index can reject nonmatching names without visiting the - // heap. The materialized candidate query plans for a larger key batch: - // a small LIMIT otherwise assumes rare matches occur early in name order - // and can favor scanning most of the ordered index over a selective GIN - // lookup. The outer LIMIT still defines the exact page before payload fetch. - // PostgreSQL 17 can stop the ordered CTE scan after the requested rows. + // Select the page using only its keys before fetching JSON payloads. payload := "s.value" if len(prepared) > 0 && prepared[0] { payload = listingPayload("s.") } - // Incremental sync needs only the exact page. Its all-status covering index - // includes deleted versions, which consumers need to remove local records. - // Retain the larger planning boundary only for substring searches. - candidateLimit := fmt.Sprintf("$%d", limitArg) - if filter.SubstringName != nil { - candidateLimit = fmt.Sprintf("GREATEST($%d, 8192)", limitArg) - } + if filter.UpdatedSince != nil && filter.SubstringName == nil { + // Probe a bounded set in timestamp order. If it contains every matching + // key, sort just that set; otherwise fetch the page in name order. + // Both paths use the complete predicate in one statement/snapshot. + // The exact page limit and late payload lookup remain unchanged. + // This bounds returned probe keys, not all timestamp-index entries visited. + return fmt.Sprintf(` + WITH recent AS MATERIALIZED ( + SELECT server_name, version FROM servers + %s + ORDER BY updated_at DESC + LIMIT 129 + ), choice AS MATERIALIZED ( + SELECT count(*) <= 128 AS small FROM recent + ), candidates AS MATERIALIZED ( + (SELECT server_name, version FROM recent + WHERE (SELECT small FROM choice) + ORDER BY server_name, version LIMIT $%d) + UNION ALL + (SELECT server_name, version FROM servers + %s AND NOT (SELECT small FROM choice) + ORDER BY server_name, version LIMIT $%d) + ) + SELECT s.server_name, s.version, s.status, s.status_changed_at, s.status_message, + s.published_at, s.updated_at, s.is_latest, %s + FROM ( + SELECT server_name, version FROM candidates + ORDER BY server_name, version LIMIT $%d + ) page + JOIN servers s USING (server_name, version) + ORDER BY s.server_name, s.version + `, whereClause, limitArg, whereClause, limitArg, payload, limitArg) + } + // For substring searches, a small LIMIT can favor scanning most of the + // ordered index over a selective GIN lookup. Plan for a larger key batch; + // the outer LIMIT still defines the exact page before payload fetch. + candidateLimit := fmt.Sprintf("GREATEST($%d, 8192)", limitArg) return fmt.Sprintf(` WITH candidates AS MATERIALIZED ( SELECT server_name, version FROM servers @@ -262,10 +286,10 @@ func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limit } func filteredQueryArgs(filter *ServerFilter, args []any) []any { - if filter != nil && filter.SubstringName != nil { - // Search selectivity varies by orders of magnitude. Use an unnamed + if filter != nil && (filter.SubstringName != nil || filter.UpdatedSince != nil) { + // Search and incremental-sync selectivity vary greatly. Use an unnamed // extended-protocol query so PostgreSQL plans for these parameter values - // instead of reusing one generic plan for every term. + // instead of reusing one generic plan for every term, cutoff, and cursor. return append([]any{pgx.QueryExecModeExec}, args...) } return args diff --git a/internal/database/updated_indexes_internal_test.go b/internal/database/updated_indexes_internal_test.go new file mode 100644 index 000000000..b18fbaabe --- /dev/null +++ b/internal/database/updated_indexes_internal_test.go @@ -0,0 +1,80 @@ +package database + +import ( + "context" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +func TestUpdatedListingIndexMigration(t *testing.T) { + for _, mode := range []string{"fresh", "upgrade", "prebuilt", "wrong-definition"} { + t.Run(mode, func(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + conn, err := db.pool.Acquire(ctx) + require.NoError(t, err) + defer conn.Release() + migrator := NewMigrator(conn.Conn()) + migrations, err := migrator.loadMigrations() + require.NoError(t, err) + var migration Migration + for _, candidate := range migrations { + if candidate.Version == 20 { + migration = candidate + } + } + require.Equal(t, 20, migration.Version) + if mode != "fresh" { + _, err = conn.Exec(ctx, ` + DROP INDEX idx_servers_updated_name_cover; + DROP INDEX idx_servers_updated_time_cover; + CREATE INDEX idx_servers_updated_page_cover ON servers(server_name,version) INCLUDE(updated_at,is_latest,status); + CREATE INDEX idx_servers_updated_at ON servers(updated_at DESC); + DELETE FROM schema_migrations WHERE version=20; + `) + require.NoError(t, err) + switch mode { + case "prebuilt": + // Concurrent builds must be separate autocommit statements. + _, err = conn.Exec(ctx, `CREATE INDEX CONCURRENTLY idx_servers_updated_name_cover ON servers(server_name,version,updated_at) INCLUDE(is_latest,status)`) + require.NoError(t, err) + _, err = conn.Exec(ctx, `CREATE INDEX CONCURRENTLY idx_servers_updated_time_cover ON servers(updated_at DESC,server_name,version)`) + require.NoError(t, err) + case "wrong-definition": + // The old INCLUDE layout must not silently satisfy the new index name. + _, err = conn.Exec(ctx, `CREATE INDEX idx_servers_updated_name_cover ON servers(server_name,version) INCLUDE(updated_at,is_latest,status)`) + require.NoError(t, err) + } + err = migrator.applyMigration(ctx, migration) + if mode == "wrong-definition" { + require.ErrorContains(t, err, "expected definition") + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_page_cover", true) + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_at", true) + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_time_cover", false) + var applied bool + require.NoError(t, conn.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version=20)").Scan(&applied)) + require.False(t, applied) + return + } + require.NoError(t, err) + } + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_page_cover", false) + assertUpdatedIndexExists(t, conn.Conn(), "idx_servers_updated_at", false) + for _, name := range []string{"idx_servers_updated_name_cover", "idx_servers_updated_time_cover"} { + var usable bool + require.NoError(t, conn.QueryRow(ctx, `SELECT indisvalid AND indisready AND indnkeyatts=3 FROM pg_index WHERE indexrelid=to_regclass($1)`, name).Scan(&usable)) + require.True(t, usable) + } + require.NoError(t, migrator.Migrate(ctx)) + }) + } +} + +func assertUpdatedIndexExists(t *testing.T, conn *pgx.Conn, name string, expected bool) { + t.Helper() + var exists bool + require.NoError(t, conn.QueryRow(context.Background(), "SELECT to_regclass($1) IS NOT NULL", name).Scan(&exists)) + require.Equal(t, expected, exists, name) +} diff --git a/internal/database/updated_page_test.go b/internal/database/updated_page_test.go index bdc8af738..e46d08a8a 100644 --- a/internal/database/updated_page_test.go +++ b/internal/database/updated_page_test.go @@ -103,6 +103,70 @@ func TestUpdatedPagePreservesWatermarkPaginationAndPayloads(t *testing.T) { } } +// Exercise both selection branches, including exact probe-boundary counts, +// using an independently constructed expected sequence and full pagination. +func TestUpdatedPageProbeBoundary(t *testing.T) { + db := database.NewTestDB(t) + ctx := context.Background() + epoch := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + yes, no := true, false + version := "1.0.0" + var fixture []*apiv0.ServerResponse + for i := 0; i < 260; i++ { + status := model.StatusActive + if i%3 == 0 { + status = model.StatusDeleted + } + stamp := epoch.Add(time.Duration(i+1) * time.Second) + r, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/boundary-%03d", i), Version: version, Description: "boundary payload"}, &apiv0.RegistryExtensions{Status: status, StatusChangedAt: stamp, PublishedAt: epoch, UpdatedAt: stamp, IsLatest: true}) + require.NoError(t, err) + fixture = append(fixture, r) + } + for _, count := range []int{0, 1, 127, 128, 129, 130, 260} { + cutoff := epoch.Add(time.Duration(260-count) * time.Second) + for _, includeDeleted := range []*bool{&yes, &no, nil} { + f := &database.ServerFilter{UpdatedSince: &cutoff, IncludeDeleted: includeDeleted, Version: &version} + want := expectedUpdatedServers(fixture, f, cutoff) + for _, limit := range []int{1, 100, 128, 129, 1000} { + var got []*apiv0.ServerResponse + cursor := "" + for page := 0; page < 262; page++ { + rows, next, err := db.ListServers(ctx, nil, f, cursor, limit) + require.NoError(t, err) + for _, r := range rows { + r.Meta.Official.PublishedAt = r.Meta.Official.PublishedAt.UTC() + r.Meta.Official.UpdatedAt = r.Meta.Official.UpdatedAt.UTC() + r.Meta.Official.StatusChangedAt = r.Meta.Official.StatusChangedAt.UTC() + } + wire, err := db.(*database.PostgreSQL).ListServersJSON(ctx, f, cursor, limit) + require.NoError(t, err) + encoded, err := json.Marshal(wire) + require.NoError(t, err) + var decoded apiv0.ServerListResponse + require.NoError(t, json.Unmarshal(encoded, &decoded)) + require.Equal(t, next, decoded.Metadata.NextCursor) + require.Len(t, decoded.Servers, len(rows)) + for i := range rows { + expectedJSON, err := json.Marshal(rows[i]) + require.NoError(t, err) + actualJSON, err := json.Marshal(decoded.Servers[i]) + require.NoError(t, err) + require.JSONEq(t, string(expectedJSON), string(actualJSON)) + } + got = append(got, rows...) + if next == "" { + break + } + require.NotEqual(t, cursor, next) + cursor = next + require.Less(t, page, 261) + } + require.Equal(t, want, got, "count=%d limit=%d", count, limit) + } + } + } +} + func expectedUpdatedServers(fixture []*apiv0.ServerResponse, filter *database.ServerFilter, cutoff time.Time) []*apiv0.ServerResponse { var want []*apiv0.ServerResponse for _, server := range fixture { From cf1e1afc1de8e57024ff2aebcb4898940d686fbf Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Wed, 18 Nov 2026 15:13:57 +0200 Subject: [PATCH 2/4] Reduce repeated planning for common incremental pages --- deploy/README.md | 9 +- internal/database/postgres.go | 46 +++++++--- .../updated_page_cache_internal_test.go | 89 +++++++++++++++++++ 3 files changed, 128 insertions(+), 16 deletions(-) create mode 100644 internal/database/updated_page_cache_internal_test.go diff --git a/deploy/README.md b/deploy/README.md index 52232ac36..df37b307e 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1189,9 +1189,12 @@ matching keys in timestamp order. With at most 128 matches it sorts that complet set into name/version order; otherwise it fetches the requested page in name order. Both branches use the full predicate and one statement snapshot, with payloads fetched only for the selected page. Timestamp order never becomes the -public pagination order. Incremental queries use per-execution planning because -cutoff and cursor selectivity vary; this must be evaluated with the query and -index combination, not as a general recommendation to disable prepared plans. +public pagination order. Plain incremental queries with page sizes 1 or 100 use +cached statements with the validated page size embedded in the query; cutoffs and +cursors remain parameters. The exact page bound lets PostgreSQL consider reusing +a generic plan instead of repeatedly planning the common shapes. Additional +filters and other page sizes retain per-execution planning. This is specific to +the adaptive query and replacement indexes, not a global plan-cache setting. The probe bounds returned keys, not all index entries examined. Very large recent histories, sparse cursors and different filter mixes still need measurement. diff --git a/internal/database/postgres.go b/internal/database/postgres.go index 3bef0be24..bcc72c764 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -220,7 +220,7 @@ func addCursorCondition(cursor string, argIndex int) (string, []any, int) { } // filteredPageQuery fetches payloads only after selecting a filtered page's keys. -func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limitArg int, prepared ...bool) string { +func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limitExpression string, prepared ...bool) string { if filter == nil || (filter.SubstringName == nil && filter.UpdatedSince == nil) { return ordinary } @@ -246,26 +246,26 @@ func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limit ), candidates AS MATERIALIZED ( (SELECT server_name, version FROM recent WHERE (SELECT small FROM choice) - ORDER BY server_name, version LIMIT $%d) + ORDER BY server_name, version LIMIT %s) UNION ALL (SELECT server_name, version FROM servers %s AND NOT (SELECT small FROM choice) - ORDER BY server_name, version LIMIT $%d) + ORDER BY server_name, version LIMIT %s) ) SELECT s.server_name, s.version, s.status, s.status_changed_at, s.status_message, s.published_at, s.updated_at, s.is_latest, %s FROM ( SELECT server_name, version FROM candidates - ORDER BY server_name, version LIMIT $%d + ORDER BY server_name, version LIMIT %s ) page JOIN servers s USING (server_name, version) ORDER BY s.server_name, s.version - `, whereClause, limitArg, whereClause, limitArg, payload, limitArg) + `, whereClause, limitExpression, whereClause, limitExpression, payload, limitExpression) } // For substring searches, a small LIMIT can favor scanning most of the // ordered index over a selective GIN lookup. Plan for a larger key batch; // the outer LIMIT still defines the exact page before payload fetch. - candidateLimit := fmt.Sprintf("GREATEST($%d, 8192)", limitArg) + candidateLimit := fmt.Sprintf("GREATEST(%s, 8192)", limitExpression) return fmt.Sprintf(` WITH candidates AS MATERIALIZED ( SELECT server_name, version FROM servers @@ -278,11 +278,19 @@ func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limit FROM ( SELECT server_name, version FROM candidates ORDER BY server_name, version - LIMIT $%d + LIMIT %s ) page JOIN servers s USING (server_name, version) ORDER BY s.server_name, s.version - `, whereClause, candidateLimit, payload, limitArg) + `, whereClause, candidateLimit, payload, limitExpression) +} + +// cacheUpdatedPage bounds the extra statement-cache shapes to the two common +// page sizes. Other incremental filters retain parameter-specific planning. +func cacheUpdatedPage(filter *ServerFilter, limit int) bool { + return filter != nil && filter.UpdatedSince != nil && + filter.SubstringName == nil && filter.Name == nil && filter.Version == nil && + filter.IsLatest == nil && filter.RemoteURL == nil && (limit == 1 || limit == 100) } func filteredQueryArgs(filter *ServerFilter, args []any) []any { @@ -375,16 +383,28 @@ func (db *PostgreSQL) listServerRows( if prepared { payload = listingPayload("") } + cachePage := cacheUpdatedPage(filter, limit) + limitExpression := fmt.Sprintf("$%d", argIndex) + if cachePage { + // Only the validated constants 1 and 100 are embedded. Cutoffs and + // cursors remain parameters. This gives generic plans an exact page + // bound without forcing generic planning for other query shapes. + limitExpression = fmt.Sprintf("%d", limit) + } query := fmt.Sprintf(` SELECT server_name, version, status, status_changed_at, status_message, published_at, updated_at, is_latest, %s FROM servers %s ORDER BY server_name, version - LIMIT $%d - `, payload, whereClause, argIndex) - query = filteredPageQuery(filter, query, whereClause, argIndex, prepared) - args = append(args, limit) - args = filteredQueryArgs(filter, args) + LIMIT %s + `, payload, whereClause, limitExpression) + query = filteredPageQuery(filter, query, whereClause, limitExpression, prepared) + if cachePage { + args = append([]any{pgx.QueryExecModeCacheStatement}, args...) + } else { + args = append(args, limit) + args = filteredQueryArgs(filter, args) + } rows, err := db.getExecutor(tx).Query(ctx, query, args...) if err != nil { diff --git a/internal/database/updated_page_cache_internal_test.go b/internal/database/updated_page_cache_internal_test.go new file mode 100644 index 000000000..bcde288be --- /dev/null +++ b/internal/database/updated_page_cache_internal_test.go @@ -0,0 +1,89 @@ +package database + +import ( + "context" + "fmt" + "testing" + "time" + + apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0" + "github.com/modelcontextprotocol/registry/pkg/model" + "github.com/stretchr/testify/require" +) + +func TestUpdatedPageCachePreservesChangingWatermarksAndBounds(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + epoch := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + for i := 1; i <= 140; i++ { + stamp := epoch.Add(time.Duration(i) * time.Second) + _, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/cached-%03d", i), Version: "1.0.0", Description: "cached page fixture"}, &apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: epoch, UpdatedAt: stamp, StatusChangedAt: stamp, IsLatest: true}) + require.NoError(t, err) + } + // Keep one connection so each query shape crosses the prepared-plan threshold. + conn, err := db.pool.Acquire(ctx) + require.NoError(t, err) + defer conn.Release() + tx, err := conn.Begin(ctx) + require.NoError(t, err) + defer func() { require.NoError(t, tx.Rollback(ctx)) }() + yes := true + for round := 0; round < 3; round++ { + for _, count := range []int{1, 128, 129, 140, 0, 21} { + cutoff := epoch.Add(time.Duration(140-count) * time.Second) + filter := &ServerFilter{UpdatedSince: &cutoff, IncludeDeleted: &yes} + for _, limit := range []int{1, 100} { + for _, after := range []int{0, 51} { + cursor := "" + if after > 0 { + cursor = fmt.Sprintf("com.example/cached-%03d:1.0.0", after) + } + rows, _, err := db.ListServers(ctx, tx, filter, cursor, limit) + require.NoError(t, err) + var want []string + for i := 1; i <= 140 && len(want) < limit; i++ { + if i > 140-count && i > after { + want = append(want, fmt.Sprintf("com.example/cached-%03d", i)) + } + } + var got []string + for _, row := range rows { + got = append(got, row.Server.Name) + } + require.Equal(t, want, got, "round=%d count=%d limit=%d after=%d", round, count, limit, after) + } + } + } + } + var cached int + require.NoError(t, tx.QueryRow(ctx, "SELECT count(*) FROM pg_prepared_statements WHERE statement ~ '^[[:space:]]*WITH recent AS MATERIALIZED'").Scan(&cached)) + require.Equal(t, 4, cached, "two page sizes, with and without a compound cursor") + var parameterizedLimit int + require.NoError(t, tx.QueryRow(ctx, "SELECT count(*) FROM pg_prepared_statements WHERE statement ~ '^[[:space:]]*WITH recent AS MATERIALIZED' AND statement LIKE '%LIMIT $%'").Scan(¶meterizedLimit)) + require.Zero(t, parameterizedLimit) + + // Changes are visible without reconnecting or clearing the cached statements. + _, err = tx.Exec(ctx, "UPDATE servers SET updated_at=$1 WHERE server_name LIKE 'com.example/cached-%'", epoch) + require.NoError(t, err) + rows, _, err := db.ListServers(ctx, tx, &ServerFilter{UpdatedSince: &epoch, IncludeDeleted: &yes}, "", 100) + require.NoError(t, err) + require.Empty(t, rows) + + // Keep additional filters and uncommon sizes on their existing custom path. + version, name, search, remote := "1.0.0", "com.example/cached-001", "cached", "https://example.invalid/mcp" + for _, filter := range []*ServerFilter{ + {UpdatedSince: &epoch, Version: &version}, {UpdatedSince: &epoch, Name: &name}, + {UpdatedSince: &epoch, SubstringName: &search}, {UpdatedSince: &epoch, IsLatest: &yes}, + {UpdatedSince: &epoch, RemoteURL: &remote}, + } { + _, _, err = db.ListServers(ctx, tx, filter, "", 100) + require.NoError(t, err) + } + for _, limit := range []int{0, 10, 128, 1000} { + _, _, err = db.ListServers(ctx, tx, &ServerFilter{UpdatedSince: &epoch}, "", limit) + require.NoError(t, err) + } + var after int + require.NoError(t, tx.QueryRow(ctx, "SELECT count(*) FROM pg_prepared_statements WHERE statement ~ '^[[:space:]]*WITH recent AS MATERIALIZED'").Scan(&after)) + require.Equal(t, cached, after) +} From b888b8e62a6f5947695cd8fda84b84aa42721e3f Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Wed, 18 Nov 2026 17:28:43 +0200 Subject: [PATCH 3/4] Widen the common incremental page probe to 512 keys --- deploy/README.md | 11 +++++++---- internal/database/postgres.go | 12 +++++++++--- .../database/updated_page_cache_internal_test.go | 10 +++++----- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index df37b307e..eba58e696 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1184,10 +1184,13 @@ Migration 020 replaces migration 019's included-column timestamp index with `idx_servers_updated_time_cover` on `(updated_at DESC, server_name, version)`. Both include deleted versions needed by incremental-sync clients. -For `updated_since` without substring search, the query first probes up to 129 -matching keys in timestamp order. With at most 128 matches it sorts that complete -set into name/version order; otherwise it fetches the requested page in name -order. Both branches use the full predicate and one statement snapshot, with +For plain `updated_since` queries with `limit=100`, the query first probes up to +513 matching keys in timestamp order. With at most 512 matches it sorts that +complete set into name/version order; otherwise it fetches the requested page in +name order. Other incremental shapes without substring search retain the smaller +129-key probe and 128-match boundary. The larger probe avoids switching to a +costly name scan for moderately sparse pages, but adds work for larger histories. +Both branches use the full predicate and one statement snapshot, with payloads fetched only for the selected page. Timestamp order never becomes the public pagination order. Plain incremental queries with page sizes 1 or 100 use cached statements with the validated page size embedded in the query; cutoffs and diff --git a/internal/database/postgres.go b/internal/database/postgres.go index bcc72c764..d9f971f99 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -230,6 +230,12 @@ func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limit payload = listingPayload("s.") } if filter.UpdatedSince != nil && filter.SubstringName == nil { + probe := 128 + // The cached 100-row shape uses a larger probe for sparse histories. + // Keep the smaller bound for limit=1 and parameter-specific queries. + if limitExpression == "100" { + probe = 512 + } // Probe a bounded set in timestamp order. If it contains every matching // key, sort just that set; otherwise fetch the page in name order. // Both paths use the complete predicate in one statement/snapshot. @@ -240,9 +246,9 @@ func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limit SELECT server_name, version FROM servers %s ORDER BY updated_at DESC - LIMIT 129 + LIMIT %d ), choice AS MATERIALIZED ( - SELECT count(*) <= 128 AS small FROM recent + SELECT count(*) <= %d AS small FROM recent ), candidates AS MATERIALIZED ( (SELECT server_name, version FROM recent WHERE (SELECT small FROM choice) @@ -260,7 +266,7 @@ func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limit ) page JOIN servers s USING (server_name, version) ORDER BY s.server_name, s.version - `, whereClause, limitExpression, whereClause, limitExpression, payload, limitExpression) + `, whereClause, probe+1, probe, limitExpression, whereClause, limitExpression, payload, limitExpression) } // For substring searches, a small LIMIT can favor scanning most of the // ordered index over a selective GIN lookup. Plan for a larger key batch; diff --git a/internal/database/updated_page_cache_internal_test.go b/internal/database/updated_page_cache_internal_test.go index bcde288be..953cf8da3 100644 --- a/internal/database/updated_page_cache_internal_test.go +++ b/internal/database/updated_page_cache_internal_test.go @@ -15,7 +15,7 @@ func TestUpdatedPageCachePreservesChangingWatermarksAndBounds(t *testing.T) { db := NewTestDB(t).(*PostgreSQL) ctx := context.Background() epoch := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) - for i := 1; i <= 140; i++ { + for i := 1; i <= 540; i++ { stamp := epoch.Add(time.Duration(i) * time.Second) _, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/cached-%03d", i), Version: "1.0.0", Description: "cached page fixture"}, &apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: epoch, UpdatedAt: stamp, StatusChangedAt: stamp, IsLatest: true}) require.NoError(t, err) @@ -29,8 +29,8 @@ func TestUpdatedPageCachePreservesChangingWatermarksAndBounds(t *testing.T) { defer func() { require.NoError(t, tx.Rollback(ctx)) }() yes := true for round := 0; round < 3; round++ { - for _, count := range []int{1, 128, 129, 140, 0, 21} { - cutoff := epoch.Add(time.Duration(140-count) * time.Second) + for _, count := range []int{1, 128, 129, 511, 512, 513, 540, 0, 21} { + cutoff := epoch.Add(time.Duration(540-count) * time.Second) filter := &ServerFilter{UpdatedSince: &cutoff, IncludeDeleted: &yes} for _, limit := range []int{1, 100} { for _, after := range []int{0, 51} { @@ -41,8 +41,8 @@ func TestUpdatedPageCachePreservesChangingWatermarksAndBounds(t *testing.T) { rows, _, err := db.ListServers(ctx, tx, filter, cursor, limit) require.NoError(t, err) var want []string - for i := 1; i <= 140 && len(want) < limit; i++ { - if i > 140-count && i > after { + for i := 1; i <= 540 && len(want) < limit; i++ { + if i > 540-count && i > after { want = append(want, fmt.Sprintf("com.example/cached-%03d", i)) } } From 5dbdbe8d0231b331672bc174ac0db1c8c3e20720 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Wed, 18 Nov 2026 22:04:39 +0200 Subject: [PATCH 4/4] Reuse bounded strategy hints for larger incremental histories --- deploy/README.md | 12 +- internal/database/postgres.go | 62 ++--- internal/database/updated_page.go | 221 ++++++++++++++++++ .../updated_page_hints_internal_test.go | 210 +++++++++++++++++ .../updated_page_strategy_internal_test.go | 115 +++++++++ internal/database/updated_page_test.go | 6 + 6 files changed, 579 insertions(+), 47 deletions(-) create mode 100644 internal/database/updated_page.go create mode 100644 internal/database/updated_page_hints_internal_test.go create mode 100644 internal/database/updated_page_strategy_internal_test.go diff --git a/deploy/README.md b/deploy/README.md index eba58e696..9a7ca3a52 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1189,7 +1189,17 @@ For plain `updated_since` queries with `limit=100`, the query first probes up to complete set into name/version order; otherwise it fetches the requested page in name order. Other incremental shapes without substring search retain the smaller 129-key probe and 128-match boundary. The larger probe avoids switching to a -costly name scan for moderately sparse pages, but adds work for larger histories. +costly name scan for moderately sparse pages. To avoid repeatedly paying for that +probe on large histories, each app samples one in 16 eligible plain 100-row +requests and retains up to 32 query-strategy hints for five seconds. A hint +records a timestamp/name range containing more than 512 matches. Requests +covered by that range can skip the probe; PostgreSQL verifies cursor ordering +behind a materialized guard so this decision does not defeat generic-plan reuse. +Hints are separated by deletion filtering and are not used in caller-owned +transactions or additional filter shapes. They contain no response data: writes +can make a hint suboptimal until it expires, but every request still applies the +complete predicate to current data. Sampled queries return an internal marker +row that is removed before pagination, serialization and returned-row metrics. Both branches use the full predicate and one statement snapshot, with payloads fetched only for the selected page. Timestamp order never becomes the public pagination order. Plain incremental queries with page sizes 1 or 100 use diff --git a/internal/database/postgres.go b/internal/database/postgres.go index d9f971f99..3e3a25a3a 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -22,9 +22,10 @@ import ( // PostgreSQL is an implementation of the Database interface using PostgreSQL type PostgreSQL struct { - pool *pgxpool.Pool - poolMetrics metric.Registration - listingMetrics metric.Int64Counter + updatedPageHints updatedPageHints + pool *pgxpool.Pool + poolMetrics metric.Registration + listingMetrics metric.Int64Counter } // Executor is an interface for executing queries (satisfied by both pgx.Tx and pgxpool.Pool) @@ -220,53 +221,17 @@ func addCursorCondition(cursor string, argIndex int) (string, []any, int) { } // filteredPageQuery fetches payloads only after selecting a filtered page's keys. -func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limitExpression string, prepared ...bool) string { +func filteredPageQuery(filter *ServerFilter, ordinary, whereClause string, limitExpression string, prepared bool, strategy updatedPageStrategy) string { if filter == nil || (filter.SubstringName == nil && filter.UpdatedSince == nil) { return ordinary } // Select the page using only its keys before fetching JSON payloads. payload := "s.value" - if len(prepared) > 0 && prepared[0] { + if prepared { payload = listingPayload("s.") } if filter.UpdatedSince != nil && filter.SubstringName == nil { - probe := 128 - // The cached 100-row shape uses a larger probe for sparse histories. - // Keep the smaller bound for limit=1 and parameter-specific queries. - if limitExpression == "100" { - probe = 512 - } - // Probe a bounded set in timestamp order. If it contains every matching - // key, sort just that set; otherwise fetch the page in name order. - // Both paths use the complete predicate in one statement/snapshot. - // The exact page limit and late payload lookup remain unchanged. - // This bounds returned probe keys, not all timestamp-index entries visited. - return fmt.Sprintf(` - WITH recent AS MATERIALIZED ( - SELECT server_name, version FROM servers - %s - ORDER BY updated_at DESC - LIMIT %d - ), choice AS MATERIALIZED ( - SELECT count(*) <= %d AS small FROM recent - ), candidates AS MATERIALIZED ( - (SELECT server_name, version FROM recent - WHERE (SELECT small FROM choice) - ORDER BY server_name, version LIMIT %s) - UNION ALL - (SELECT server_name, version FROM servers - %s AND NOT (SELECT small FROM choice) - ORDER BY server_name, version LIMIT %s) - ) - SELECT s.server_name, s.version, s.status, s.status_changed_at, s.status_message, - s.published_at, s.updated_at, s.is_latest, %s - FROM ( - SELECT server_name, version FROM candidates - ORDER BY server_name, version LIMIT %s - ) page - JOIN servers s USING (server_name, version) - ORDER BY s.server_name, s.version - `, whereClause, probe+1, probe, limitExpression, whereClause, limitExpression, payload, limitExpression) + return updatedPageQuery(whereClause, limitExpression, payload, prepared, strategy) } // For substring searches, a small LIMIT can favor scanning most of the // ordered index over a selective GIN lookup. Plan for a larger key batch; @@ -372,6 +337,7 @@ func (db *PostgreSQL) listServerRows( whereConditions, args, argIndex := buildFilterConditions(filter, argIndex) // Add cursor pagination + cursorArgIndex := argIndex cursorCondition, cursorArgs, argIndex := addCursorCondition(cursor, argIndex) if cursorCondition != "" { whereConditions = append(whereConditions, cursorCondition) @@ -404,7 +370,8 @@ func (db *PostgreSQL) listServerRows( ORDER BY server_name, version LIMIT %s `, payload, whereClause, limitExpression) - query = filteredPageQuery(filter, query, whereClause, limitExpression, prepared) + strategy, args := db.chooseUpdatedPageStrategy(tx, filter, limit, cursorArgs, cursorArgIndex, argIndex, args) + query = filteredPageQuery(filter, query, whereClause, limitExpression, prepared, strategy) if cachePage { args = append([]any{pgx.QueryExecModeCacheStatement}, args...) } else { @@ -416,7 +383,7 @@ func (db *PostgreSQL) listServerRows( if err != nil { return nil, fmt.Errorf("failed to query servers: %w", err) } - return readListingRows(ctx, rows, diagnostic, prepared) + return db.readUpdatedPageRows(ctx, rows, diagnostic, prepared, strategy, filter) } type listingRow struct { @@ -426,7 +393,7 @@ type listingRow struct { prepared bool } -func readListingRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.ListingDiagnostic, prepared bool) ([]listingRow, error) { +func readListingRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.ListingDiagnostic, prepared bool, observation *updatedPageObservation) ([]listingRow, error) { defer rows.Close() // Copy the bounded page before decoding JSON. Exhausting/closing pooled rows @@ -436,7 +403,6 @@ func readListingRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.L var page []listingRow diagnostic.Phase("row_fetch") for rows.Next() { - diagnostic.RowReceived() diagnostic.Phase("row_fetch") if err := ctx.Err(); err != nil { return nil, err @@ -453,6 +419,10 @@ func readListingRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.L if err := rows.Scan(dest...); err != nil { return nil, fmt.Errorf("failed to scan server row: %w", err) } + if observation != nil && observation.consume(row) { + continue + } + diagnostic.RowReceived() meta.Status = model.Status(status) diagnostic.AddBytes(len(row.value)) page = append(page, row) diff --git a/internal/database/updated_page.go b/internal/database/updated_page.go new file mode 100644 index 000000000..096ca716b --- /dev/null +++ b/internal/database/updated_page.go @@ -0,0 +1,221 @@ +package database + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/modelcontextprotocol/registry/internal/telemetry" +) + +const ( + updatedPageProbe = 512 + updatedPageHintLimit = 32 + updatedPageHintLifetime = 5 * time.Second + updatedPageObservationInterval = 16 +) + +// updatedPageHints remembers query strategies, never response rows. A hint says +// a recent probe found >512 matching rows with minimum timestamp/name boundaries. +// Writes can invalidate that observation immediately: a stale hint may choose a +// slower name-ordered scan, but that scan still applies the complete predicate. +// Expiry bounds this performance risk; the fixed capacity bounds per-app memory. +// The zero value is ready to use. A PostgreSQL instance must not be copied. +type updatedPageHints struct { + mu sync.Mutex + requests uint64 + entries []updatedPageHint +} + +type updatedPageHint struct { + expires time.Time + before time.Time + nameBefore string + includeDeleted bool +} + +type updatedPageStrategy struct { + observe bool + skipProbe bool + nameBefore string + cursorGuard string // SQL over bound parameters, never interpolated cursor data. +} + +func (h *updatedPageHints) choose(now, since time.Time, cursorName string, includeDeleted bool) updatedPageStrategy { + h.mu.Lock() + defer h.mu.Unlock() + h.requests++ + strategy := updatedPageStrategy{observe: h.requests%updatedPageObservationInterval == 1} + for _, hint := range h.entries { + if hint.includeDeleted != includeDeleted || !now.Before(hint.expires) || !since.Before(hint.before) { + continue + } + // This screen can miss opportunities under a different DB collation. + // SQL rechecks strict name ordering before skipping a cursor-page probe. + if cursorName != "" && cursorName >= hint.nameBefore { + continue + } + if !strategy.skipProbe || hint.nameBefore > strategy.nameBefore { + strategy.skipProbe = true + strategy.nameBefore = hint.nameBefore + } + } + return strategy +} + +func (h *updatedPageHints) remember(now time.Time, observation updatedPageObservation, includeDeleted bool) { + h.mu.Lock() + defer h.mu.Unlock() + // Do not refresh an existing hint's lifetime without a fresh probe. A + // covered sampled query has no probe rows and does not produce a hint. + live := h.entries[:0] + for _, hint := range h.entries { + if now.Before(hint.expires) { + live = append(live, hint) + } + } + h.entries = live + for _, hint := range h.entries { + if hint.includeDeleted == includeDeleted && hint.before.Equal(observation.before) && hint.nameBefore == observation.nameBefore { + return + } + } + entry := updatedPageHint{expires: now.Add(updatedPageHintLifetime), before: observation.before, nameBefore: observation.nameBefore, includeDeleted: includeDeleted} + if len(h.entries) < updatedPageHintLimit { + h.entries = append(h.entries, entry) + return + } + oldest := 0 + for i := range h.entries { + if h.entries[i].expires.Before(h.entries[oldest].expires) { + oldest = i + } + } + h.entries[oldest] = entry +} + +// updatedPageObservation is returned in one private marker row on sampled +// queries. Empty server names are forbidden by the schema, so the marker cannot +// collide with a stored row. Reusing the row layout avoids widening every page +// row; these fields never reach API serialization or returned-row telemetry. +type updatedPageObservation struct { + large bool + before time.Time + nameBefore string +} + +func (o *updatedPageObservation) consume(row listingRow) bool { + if row.name != "" { + return false + } + o.large = row.metadata.IsLatest + o.before = row.metadata.UpdatedAt + if row.metadata.StatusMessage != nil { + o.nameBefore = *row.metadata.StatusMessage + } + return true +} + +// chooseUpdatedPageStrategy is limited to the common plain 100-row shape. +// Transaction-owned reads and additional filters never consult or populate hints. +func (db *PostgreSQL) chooseUpdatedPageStrategy(tx pgx.Tx, filter *ServerFilter, limit int, cursorArgs []any, cursorArgIndex, nextArg int, args []any) (updatedPageStrategy, []any) { + if tx != nil || limit != 100 || !cacheUpdatedPage(filter, limit) { + return updatedPageStrategy{}, args + } + includeDeleted := filter.IncludeDeleted != nil && *filter.IncludeDeleted + cursorName := "" + if len(cursorArgs) > 0 { + cursorName = cursorArgs[0].(string) + } + strategy := db.updatedPageHints.choose(time.Now(), *filter.UpdatedSince, cursorName, includeDeleted) + if strategy.skipProbe && len(cursorArgs) > 0 { + // PostgreSQL verifies ordering with server_name's default collation. + strategy.cursorGuard = fmt.Sprintf("($%d::text < $%d::text)", cursorArgIndex, nextArg) + args = append(args, strategy.nameBefore) + } + return strategy, args +} + +func (db *PostgreSQL) readUpdatedPageRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.ListingDiagnostic, prepared bool, strategy updatedPageStrategy, filter *ServerFilter) ([]listingRow, error) { + var observation *updatedPageObservation + if strategy.observe { + observation = &updatedPageObservation{} + } + page, err := readListingRows(ctx, rows, diagnostic, prepared, observation) + if err == nil && observation != nil && observation.large { + db.updatedPageHints.remember(time.Now(), *observation, filter.IncludeDeleted != nil && *filter.IncludeDeleted) + } + return page, err +} + +// updatedPageQuery selects keys before fetching payloads. A complete bounded +// timestamp probe is cheap for sparse histories; a name-ordered scan avoids +// sorting a large history. Both branches use one statement snapshot and the full +// predicate. The probe bounds returned keys, not every index entry visited. +func updatedPageQuery(whereClause, limitExpression, payload string, prepared bool, strategy updatedPageStrategy) string { + probe := 128 + if limitExpression == "100" { + probe = updatedPageProbe + } + prefix, covered := "", "FALSE" + if strategy.skipProbe { + covered = "TRUE" + if strategy.cursorGuard != "" { + // Constant folding otherwise makes early custom plans artificially + // cheap and can prevent PostgreSQL from reusing a generic plan. + prefix = "hint_guard AS MATERIALIZED (SELECT " + strategy.cursorGuard + " AS covers_page), " + covered = "(SELECT covers_page FROM hint_guard)" + } + } + columns, observation := "", "" + probeWhere := whereClause + choice := fmt.Sprintf("count(*) <= %d", probe) + if strategy.observe || strategy.skipProbe { + probeWhere += " AND NOT (" + covered + ")" + choice += " AND NOT (" + covered + ")" + } + if strategy.observe { + columns = ", updated_at" + observation = fmt.Sprintf(", count(*) > %d AS observed_large", probe) + } + ctes := fmt.Sprintf(`WITH %srecent AS MATERIALIZED ( + SELECT server_name, version%s FROM servers + %s + ORDER BY updated_at DESC LIMIT %d + ), choice AS MATERIALIZED ( + SELECT %s AS small%s FROM recent + ), candidates AS MATERIALIZED ( + (SELECT server_name, version FROM recent + WHERE (SELECT small FROM choice) + ORDER BY server_name, version LIMIT %s) + UNION ALL + (SELECT server_name, version FROM servers + %s AND NOT (SELECT small FROM choice) + ORDER BY server_name, version LIMIT %s) + )`, prefix, columns, probeWhere, probe+1, choice, observation, limitExpression, whereClause, limitExpression) + page := fmt.Sprintf(`SELECT s.server_name, s.version, s.status, s.status_changed_at, s.status_message, + s.published_at, s.updated_at, s.is_latest, %s + FROM (SELECT server_name, version FROM candidates + ORDER BY server_name, version LIMIT %s) page + JOIN servers s USING (server_name, version)`, payload, limitExpression) + if !strategy.observe { + return ctes + " " + page + " ORDER BY s.server_name, s.version" + } + return ctes + ", response_rows AS (" + page + ") SELECT * FROM response_rows UNION ALL " + + updatedPageMarker(prepared) + " ORDER BY server_name, version" +} + +func updatedPageMarker(prepared bool) string { + payload := "NULL::jsonb" + if prepared { + payload = "NULL::bytea, false" + } + return `SELECT ''::text AS server_name, ''::text AS version, ''::text AS status, + 'epoch'::timestamptz AS status_changed_at, + (SELECT CASE WHEN observed_large THEN (SELECT min(server_name) FROM recent) ELSE ''::text END FROM choice) AS status_message, + 'epoch'::timestamptz AS published_at, + (SELECT CASE WHEN observed_large THEN (SELECT min(updated_at) FROM recent) ELSE 'epoch'::timestamptz END FROM choice) AS updated_at, + (SELECT observed_large FROM choice) AS is_latest, ` + payload +} diff --git a/internal/database/updated_page_hints_internal_test.go b/internal/database/updated_page_hints_internal_test.go new file mode 100644 index 000000000..be7c070ba --- /dev/null +++ b/internal/database/updated_page_hints_internal_test.go @@ -0,0 +1,210 @@ +package database + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" + + "github.com/modelcontextprotocol/registry/internal/telemetry" + apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0" + "github.com/modelcontextprotocol/registry/pkg/model" +) + +func TestUpdatedPageHintsNeverCacheRows(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + epoch := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + yes := true + for i := 0; i < 540; i++ { + _, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/hint-%03d", i), Version: "1.0.0", Description: "hint fixture"}, &apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: epoch, UpdatedAt: epoch.Add(time.Second), StatusChangedAt: epoch, IsLatest: true}) + require.NoError(t, err) + } + // The private observation marker relies on this database invariant, even + // when a writer bypasses CreateServer's own name validation. + _, invalidNameErr := db.pool.Exec(ctx, "UPDATE servers SET server_name='' WHERE server_name='com.example/hint-000'") + require.ErrorContains(t, invalidNameErr, "check_server_name_format") + f := &ServerFilter{UpdatedSince: &epoch, IncludeDeleted: &yes} + read := func() ([]apiv0.ServerResponse, error) { + raw, err := db.ListServersJSON(ctx, f, "", 100) + if err != nil { + return nil, err + } + b, err := json.Marshal(raw) + if err != nil { + return nil, err + } + var out apiv0.ServerListResponse + err = json.Unmarshal(b, &out) + return out.Servers, err + } + page, err := read() + require.NoError(t, err) + require.Len(t, page, 100) + db.updatedPageHints.mu.Lock() + require.Len(t, db.updatedPageHints.entries, 1) + for k := range db.updatedPageHints.entries { + h := db.updatedPageHints.entries[k] + h.expires = time.Now().Add(time.Hour) + db.updatedPageHints.entries[k] = h + } + db.updatedPageHints.mu.Unlock() + _, err = db.pool.Exec(ctx, "UPDATE servers SET updated_at=$1", epoch) + require.NoError(t, err) + page, err = read() + require.NoError(t, err) + require.Empty(t, page) + _, err = db.pool.Exec(ctx, "UPDATE servers SET updated_at=$1 WHERE server_name < 'com.example/hint-030'", epoch.Add(time.Second)) + require.NoError(t, err) + page, err = read() + require.NoError(t, err) + require.Len(t, page, 30) + rows, _, err := db.ListServers(ctx, nil, f, "", 100) + require.NoError(t, err) + require.Len(t, rows, 30) + // Concurrent requests exercise the advisory map while all results stay current. + var wg sync.WaitGroup + errs := make(chan error, 16) + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 10; j++ { + r, e := read() + if e != nil { + errs <- e + return + } + if len(r) != 30 { + errs <- fmt.Errorf("rows=%d", len(r)) + return + } + } + }() + } + wg.Wait() + close(errs) + for e := range errs { + require.NoError(t, e) + } + db.updatedPageHints.mu.Lock() + for k := range db.updatedPageHints.entries { + h := db.updatedPageHints.entries[k] + h.expires = time.Now().Add(-time.Second) + db.updatedPageHints.entries[k] = h + } + db.updatedPageHints.mu.Unlock() + page, err = read() + require.NoError(t, err) + require.Len(t, page, 30) + _, err = db.pool.Exec(ctx, "UPDATE servers SET updated_at=$1", epoch.Add(time.Second)) + require.NoError(t, err) + page, err = read() + require.NoError(t, err) + require.Len(t, page, 100) +} + +// A moving sync watermark and cursor must not turn an algorithm hint into cached data. +func TestUpdatedPageHintsMovingWatermarkAndBoundedKeys(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + epoch := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + stamp := epoch.Add(time.Second) + yes := true + for i := 0; i < 540; i++ { + _, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/range-%03d", i), Version: "1.0.0", Description: "range fixture"}, &apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: epoch, UpdatedAt: stamp, StatusChangedAt: epoch, IsLatest: true}) + require.NoError(t, err) + } + read := func(since time.Time, cursor string, include bool) int { + f := &ServerFilter{UpdatedSince: &since, IncludeDeleted: &include} + raw, err := db.ListServersJSON(ctx, f, cursor, 100) + require.NoError(t, err) + b, err := json.Marshal(raw) + require.NoError(t, err) + var response apiv0.ServerListResponse + require.NoError(t, json.Unmarshal(b, &response)) + return len(response.Servers) + } + require.Equal(t, 100, read(epoch, "", yes)) + require.Equal(t, 100, read(epoch.Add(time.Millisecond), "", yes)) + require.Equal(t, 100, read(epoch, "com.example/aaa", yes), "legacy name-only cursor") + require.Equal(t, 39, read(epoch, "com.example/range-500", yes)) + require.Equal(t, 0, read(stamp, "", yes)) + require.Equal(t, 0, read(stamp.Add(time.Second), "", yes)) + require.Equal(t, 39, read(epoch, "com.example/range-500:1.0.0", yes)) + _, err := db.pool.Exec(ctx, "UPDATE servers SET updated_at=$1 WHERE server_name < 'com.example/range-040'", stamp.Add(time.Second)) + require.NoError(t, err) + require.Equal(t, 40, read(stamp, "", yes)) + _, err = db.pool.Exec(ctx, "UPDATE servers SET status='deleted' WHERE server_name < 'com.example/range-520'") + require.NoError(t, err) + require.Equal(t, 20, read(epoch, "", false)) + require.Equal(t, 100, read(epoch, "", yes)) + // Arbitrary valid cursor parameters cannot grow the advisory map without bound. + for i := 0; i < 540; i++ { + require.Equal(t, 100, read(epoch, fmt.Sprintf("com.example/aaa:%06d", i), yes)) + } + db.updatedPageHints.mu.Lock() + require.LessOrEqual(t, len(db.updatedPageHints.entries), 32) + for k, h := range db.updatedPageHints.entries { + h.expires = time.Now().Add(-time.Second) + db.updatedPageHints.entries[k] = h + } + db.updatedPageHints.mu.Unlock() + for i := 0; i < 16; i++ { + require.Equal(t, 100, read(epoch, "com.example/aab:1", yes)) + } + db.updatedPageHints.mu.Lock() + require.Len(t, db.updatedPageHints.entries, 1) + db.updatedPageHints.mu.Unlock() +} + +// Exercise the actual prepared guard shape on one connection: a custom-plan +// shortcut must not prevent generic reuse as watermarks/cursors change. +func TestUpdatedPageGuardReusesPlanAndChecksCursor(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + epoch := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i := 0; i < 540; i++ { + _, err := db.CreateServer(ctx, nil, &apiv0.ServerJSON{Name: fmt.Sprintf("com.example/guard-%03d", i), Version: "1.0.0", Description: "guard fixture"}, &apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: epoch, UpdatedAt: epoch.Add(time.Second), StatusChangedAt: epoch, IsLatest: true}) + require.NoError(t, err) + } + conn, err := db.pool.Acquire(ctx) + require.NoError(t, err) + defer conn.Release() + strategy := updatedPageStrategy{skipProbe: true, cursorGuard: "($2::text < $4::text)"} + query := updatedPageQuery("WHERE updated_at > $1 AND (server_name, version) > ($2, $3)", "100", "s.value", false, strategy) + for i := 0; i < 24; i++ { + rows, err := conn.Query(ctx, query, pgx.QueryExecModeCacheStatement, epoch.Add(time.Duration(i)*time.Millisecond), "com.example/aaa", fmt.Sprint(i), "com.example/guard-000") + require.NoError(t, err) + page, err := readListingRows(ctx, rows, telemetry.ListingFromContext(ctx), false, nil) + require.NoError(t, err) + require.Len(t, page, 100) + require.Equal(t, "com.example/guard-000", page[0].name) + } + var generic, custom int + require.NoError(t, conn.QueryRow(ctx, "SELECT generic_plans, custom_plans FROM pg_prepared_statements WHERE statement = $1", query).Scan(&generic, &custom)) + require.Positive(t, generic, "guard must permit generic-plan reuse") + require.Less(t, custom, 24) + // Deliberately supply an unsuitable hint. The database guard rejects it; + // the timestamp probe still returns the sparse remaining page correctly. + rows, err := conn.Query(ctx, query, pgx.QueryExecModeCacheStatement, epoch, "com.example/guard-530", "1.0.0", "com.example/guard-000") + require.NoError(t, err) + page, err := readListingRows(ctx, rows, telemetry.ListingFromContext(ctx), false, nil) + require.NoError(t, err) + require.Len(t, page, 9) + require.Equal(t, "com.example/guard-531", page[0].name) + // An explicit transaction bypasses hints even when the cache is populated. + tx, err := conn.Begin(ctx) + require.NoError(t, err) + defer func() { require.NoError(t, tx.Rollback(ctx)) }() + db.updatedPageHints.remember(time.Now(), updatedPageObservation{large: true, before: epoch.Add(time.Second), nameBefore: "com.example/guard-000"}, false) + before := db.updatedPageHints.requests + _, _, err = db.ListServers(ctx, tx, &ServerFilter{UpdatedSince: &epoch}, "", 100) + require.NoError(t, err) + require.Equal(t, before, db.updatedPageHints.requests) +} diff --git a/internal/database/updated_page_strategy_internal_test.go b/internal/database/updated_page_strategy_internal_test.go new file mode 100644 index 000000000..144dc6d1c --- /dev/null +++ b/internal/database/updated_page_strategy_internal_test.go @@ -0,0 +1,115 @@ +package database + +import ( + "fmt" + "sync" + "testing" + "time" + + apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0" + "github.com/stretchr/testify/require" +) + +func TestUpdatedPageHintsCoverageAndExpiry(t *testing.T) { + var hints updatedPageHints + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + before := now.Add(-time.Hour) + hints.remember(now, updatedPageObservation{large: true, before: before, nameBefore: "com.example/m"}, false) + for _, tc := range []struct { + name string + at, since time.Time + cursor string + deleted, covered bool + }{ + {"first page", now, before.Add(-time.Second), "", false, true}, + {"earlier cursor", now, before.Add(-time.Second), "com.example/a", false, true}, + {"equal cursor", now, before.Add(-time.Second), "com.example/m", false, false}, + {"later cursor", now, before.Add(-time.Second), "com.example/z", false, false}, + {"equal watermark", now, before, "", false, false}, + {"later watermark", now, before.Add(time.Second), "", false, false}, + {"different deletion filter", now, before.Add(-time.Second), "", true, false}, + {"before expiry", now.Add(updatedPageHintLifetime - time.Nanosecond), before.Add(-time.Second), "", false, true}, + {"at expiry", now.Add(updatedPageHintLifetime), before.Add(-time.Second), "", false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got := hints.choose(tc.at, tc.since, tc.cursor, tc.deleted) + require.Equal(t, tc.covered, got.skipProbe) + }) + } + // Identical observations cannot indefinitely prolong an old strategy. + hints.remember(now.Add(time.Second), updatedPageObservation{large: true, before: before, nameBefore: "com.example/m"}, false) + require.Len(t, hints.entries, 1) + require.Equal(t, now.Add(updatedPageHintLifetime), hints.entries[0].expires) +} + +func TestUpdatedPageHintsBoundedSamplingAndConcurrentAccess(t *testing.T) { + var hints updatedPageHints + now := time.Now() + for i := 0; i < 64; i++ { + got := hints.choose(now, now.Add(-time.Hour), "", false) + require.Equal(t, i%updatedPageObservationInterval == 0, got.observe) + hints.remember(now.Add(time.Duration(i)*time.Millisecond), updatedPageObservation{ + large: true, before: now, nameBefore: fmt.Sprintf("com.example/%03d", i), + }, false) + } + require.Len(t, hints.entries, updatedPageHintLimit) + for _, hint := range hints.entries { + require.GreaterOrEqual(t, hint.nameBefore, "com.example/032", "oldest entries are evicted") + } + var wg sync.WaitGroup + for worker := 0; worker < 16; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 100; i++ { + hints.choose(now, now.Add(-time.Hour), "com.example/000", false) + hints.remember(now, updatedPageObservation{large: true, before: now, nameBefore: fmt.Sprintf("com.example/concurrent-%d", i)}, false) + } + }() + } + wg.Wait() + require.Len(t, hints.entries, updatedPageHintLimit) + hints.remember(now.Add(time.Minute), updatedPageObservation{large: true, before: now, nameBefore: "com.example/new"}, true) + require.Len(t, hints.entries, 1, "expired entries are removed on observation") +} + +func TestUpdatedPageObservationMarker(t *testing.T) { + stamp := time.Now() + name := "com.example/first" + ordinary := listingRow{name: name, metadata: apiv0.RegistryExtensions{UpdatedAt: stamp, IsLatest: true}} + var observation updatedPageObservation + require.False(t, observation.consume(ordinary)) + require.Equal(t, updatedPageObservation{}, observation) + marker := listingRow{metadata: apiv0.RegistryExtensions{UpdatedAt: stamp, StatusMessage: &name, IsLatest: true}} + require.True(t, observation.consume(marker)) + require.Equal(t, updatedPageObservation{large: true, before: stamp, nameBefore: name}, observation) +} + +func TestUpdatedPageStrategyScopeAndBoundParameters(t *testing.T) { + db := &PostgreSQL{} + now := time.Now() + cutoff := now.Add(-time.Hour) + name, version, search, remote, yes := "com.example/a", "1", "example", "https://example.com", true + for _, filter := range []*ServerFilter{ + nil, {}, {UpdatedSince: &cutoff, Name: &name}, {UpdatedSince: &cutoff, Version: &version}, + {UpdatedSince: &cutoff, SubstringName: &search}, {UpdatedSince: &cutoff, IsLatest: &yes}, + {UpdatedSince: &cutoff, RemoteURL: &remote}, + } { + strategy, _ := db.chooseUpdatedPageStrategy(nil, filter, 100, nil, 2, 2, nil) + require.Equal(t, updatedPageStrategy{}, strategy) + } + for _, limit := range []int{1, 10, 128, 1000} { + strategy, _ := db.chooseUpdatedPageStrategy(nil, &ServerFilter{UpdatedSince: &cutoff}, limit, nil, 2, 2, nil) + require.Equal(t, updatedPageStrategy{}, strategy) + } + require.Zero(t, db.updatedPageHints.requests) + db.updatedPageHints.remember(now, updatedPageObservation{large: true, before: now, nameBefore: "com.example/z'--"}, false) + cursor := "com.example/a' OR true --" + strategy, args := db.chooseUpdatedPageStrategy(nil, &ServerFilter{UpdatedSince: &cutoff}, 100, []any{cursor}, 2, 3, []any{cutoff, cursor}) + require.True(t, strategy.skipProbe) + require.Equal(t, "($2::text < $3::text)", strategy.cursorGuard) + require.Equal(t, []any{cutoff, cursor, "com.example/z'--"}, args) + query := updatedPageQuery("WHERE updated_at > $1 AND server_name > $2", "100", "s.value", false, strategy) + require.NotContains(t, query, cursor) + require.NotContains(t, query, strategy.nameBefore) +} diff --git a/internal/database/updated_page_test.go b/internal/database/updated_page_test.go index e46d08a8a..232fd9ed7 100644 --- a/internal/database/updated_page_test.go +++ b/internal/database/updated_page_test.go @@ -147,6 +147,12 @@ func TestUpdatedPageProbeBoundary(t *testing.T) { require.Equal(t, next, decoded.Metadata.NextCursor) require.Len(t, decoded.Servers, len(rows)) for i := range rows { + // Typed pgx timestamps can use the host timezone; compare + // the same instants regardless of the developer's locale. + meta := decoded.Servers[i].Meta.Official + meta.PublishedAt = meta.PublishedAt.UTC() + meta.UpdatedAt = meta.UpdatedAt.UTC() + meta.StatusChangedAt = meta.StatusChangedAt.UTC() expectedJSON, err := json.Marshal(rows[i]) require.NoError(t, err) actualJSON, err := json.Marshal(decoded.Servers[i])