feat(service): ADR-0059 — async query rebuild with persistent checkpoints (#650)

core/nhcore.cabal

@@ -266,6 +266,7 @@ library
     Service.QueryObjectStore
     Service.QueryObjectStore.Core
     Service.QueryObjectStore.InMemory
+    Service.QueryObjectStore.Postgres
     Service.Response
     Service.SnapshotCache
     Service.SnapshotCache.Core
@@ -286,6 +287,7 @@ library
     Service.Transport
     Service.Transport.Web
     Service.Transport.Web.BuiltinSchemas
+    Service.Transport.Web.Readiness
     Service.Transport.Web.SwaggerUI
     Service.Transport.Internal
     Service.Transport.Cli
@@ -606,6 +608,10 @@ test-suite nhcore-test-service
     Service.Command.CanAccess.PermissionFixture
     Service.Transport.Web.CommandAuthSpec
     Service.CommandExecutor.AuditLoggingSpec
+    Service.QueryObjectStore.PostgresSpec
+    Service.Query.Subscriber.ReadinessSpec
+    Service.Transport.Web.ReadinessSpec
+    Service.Application.ReadinessBuilderSpec
 
   type: exitcode-stdio-1.0
   hs-source-dirs: test-service, test

core/service/Service/Application.hs

@@ -46,6 +46,9 @@ module Service.Application (
   withHealthCheck,
   withoutHealthCheck,
   withDispatcherConfig,
+  useQueryObjectStore,
+  useReadinessEndpoint,
+  withoutReadinessEndpoint,
 
   -- * Health Check Re-export
   Web.HealthCheckConfig (..),
@@ -2270,3 +2273,29 @@ formatOAuth2ValidationError providerName provider oauthError = do
       [fmt|Provider '#{providerName}' unexpected InvalidPkceVerifier during validation: #{errMsg}|]
     InvalidRedirectUri errMsg ->
       [fmt|Provider '#{providerName}' unexpected InvalidRedirectUri during validation: #{errMsg}|]
+
+
+-- | Wire a persistent query object store backend into the application.
+--
+-- Stub — not implemented.
+useQueryObjectStore ::
+  forall config.
+  (QueryObjectStoreConfig config) =>
+  config ->
+  Application ->
+  Application
+useQueryObjectStore _ _ = panic "not implemented: Application.useQueryObjectStore"
+
+
+-- | Enable the /ready HTTP endpoint (on by default, shown for discoverability).
+--
+-- Stub — not implemented.
+useReadinessEndpoint :: Application -> Application
+useReadinessEndpoint _ = panic "not implemented: Application.useReadinessEndpoint"
+
+
+-- | Disable the /ready endpoint entirely.
+--
+-- Stub — not implemented.
+withoutReadinessEndpoint :: Application -> Application
+withoutReadinessEndpoint _ = panic "not implemented: Application.withoutReadinessEndpoint"

core/service/Service/Query/Subscriber.hs

@@ -1,9 +1,17 @@
 module Service.Query.Subscriber (
   QuerySubscriber (..),
+  Readiness (..),
+  RebuildOptions (..),
+  QueryRebuildError (..),
   new,
   start,
   stop,
   rebuildAll,
+  rebuildFrom,
+  rebuildAllAsync,
+  readinessOf,
+  readinessOfQuery,
+  rebuildOptionsDefault,
 ) where
 
 import Basics
@@ -143,6 +151,94 @@ processEventHandler subscriber rawEvent = do
     Nothing -> pass
 
 
+-- | Readiness state of a query rebuild.
+data Readiness
+  = Rebuilding
+  | Ready
+  | Failed Text
+  deriving (Eq, Show, Generic)
+
+
+-- | Options controlling a per-query rebuild.
+data RebuildOptions = RebuildOptions
+  { chunkSize :: Int
+    -- ^ Events per fetch (default: 1000).
+  , timeout :: Int
+    -- ^ Per-query rebuild timeout in seconds (default: 300).
+  , logProgress :: Bool
+    -- ^ Emit a log message after each chunk (default: True).
+  , deleteStaleHashFirst :: Bool
+    -- ^ Delete rows with mismatched query_hash before replaying (default: True).
+  }
+  deriving (Eq, Show)
+
+
+-- | Errors produced during a query rebuild.
+data QueryRebuildError
+  = RebuildTimeout Text
+    -- ^ Rebuild took longer than the configured timeout.
+  | UpdaterException Text
+    -- ^ QueryUpdater returned Err during replay.
+  | HashMismatchReplay Text
+    -- ^ Hash mismatch forced a replay, but the replay itself failed.
+  | CheckpointFetchFailed Text
+    -- ^ Could not read the resume position from the object store.
+  | EventStoreFailed Text
+    -- ^ EventStore.readFrom returned Err.
+  deriving (Eq, Show, Generic)
+
+
+-- | Default rebuild options.
+rebuildOptionsDefault :: RebuildOptions
+rebuildOptionsDefault = RebuildOptions
+  { chunkSize = 1000
+  , timeout = 300
+  , logProgress = True
+  , deleteStaleHashFirst = True
+  }
+
+
+-- | Resumable per-query rebuild from a given StreamPosition.
+--
+-- Stub — not implemented.
+rebuildFrom
+  :: QuerySubscriber
+  -> Text
+  -> StreamPosition
+  -> RebuildOptions
+  -> Task QueryRebuildError Unit
+rebuildFrom _ _ _ _ = panic "not implemented: Service.Query.Subscriber.rebuildFrom"
+
+
+-- | Spawn async rebuild for all registered queries, updating readiness states.
+--
+-- Stub — not implemented.
+rebuildAllAsync
+  :: QuerySubscriber
+  -> RebuildOptions
+  -> Task QueryRebuildError Unit
+rebuildAllAsync _ _ = panic "not implemented: Service.Query.Subscriber.rebuildAllAsync"
+
+
+-- | Fetch the aggregate readiness state of all registered queries.
+--
+-- Stub — not implemented.
+readinessOf
+  :: QuerySubscriber
+  -> Task Text Readiness
+readinessOf _ = panic "not implemented: Service.Query.Subscriber.readinessOf"
+
+
+-- | Fetch the readiness state for a specific named query.
+--
+-- Stub — not implemented.
+readinessOfQuery
+  :: QuerySubscriber
+  -> Text
+  -> Task Text (Maybe Readiness)
+readinessOfQuery _ _ = panic "not implemented: Service.Query.Subscriber.readinessOfQuery"
+
+
 -- | Process a single raw event through all relevant query updaters.
 processEvent :: QuerySubscriber -> Event Json.Value -> Task Text Unit
 processEvent subscriber rawEvent = do

core/service/Service/QueryObjectStore/Postgres.hs

@@ -0,0 +1,62 @@
+module Service.QueryObjectStore.Postgres (
+  PostgresQueryObjectStoreConfig (..),
+  QueryObjectStoreError (..),
+  createQueryObjectStore,
+) where
+
+import Basics
+import Json qualified
+import Service.QueryObjectStore.Core (QueryObjectStore (..))
+import Service.QueryObjectStore.Core qualified as Core
+import Task (Task)
+import Task qualified
+import Text (Text)
+import ToText (toText)
+
+
+-- | Errors produced by the Postgres-backed QueryObjectStore.
+data QueryObjectStoreError
+  = ConnectionFailed Text
+    -- ^ Unable to acquire Postgres connection.
+  | StatementFailed Text
+    -- ^ Hasql statement execution failed.
+  | DecodingFailed Text
+    -- ^ Hasql result decoder failed.
+  deriving (Eq, Show, Generic)
+
+
+-- | Configuration for the Postgres-backed QueryObjectStore.
+data PostgresQueryObjectStoreConfig = PostgresQueryObjectStoreConfig
+  { host :: Text
+  , databaseName :: Text
+  , user :: Text
+  , password :: Text
+  , port :: Int
+  }
+  deriving (Eq, Show)
+
+
+instance Core.QueryObjectStoreConfig PostgresQueryObjectStoreConfig where
+  createQueryObjectStore config =
+    newFromConfig config
+      |> Task.mapError toText
+
+
+-- | Create a Postgres-backed QueryObjectStore from the given config.
+--
+-- This is the public API used by tests. Delegates to the internal stub.
+createQueryObjectStore
+  :: forall query.
+     (Json.FromJSON query, Json.ToJSON query)
+  => PostgresQueryObjectStoreConfig
+  -> Task QueryObjectStoreError (QueryObjectStore query)
+createQueryObjectStore config = newFromConfig config
+
+
+-- | Internal stub. Throws a sentinel so that every test fails against this stub.
+newFromConfig
+  :: forall query.
+     (Json.FromJSON query, Json.ToJSON query)
+  => PostgresQueryObjectStoreConfig
+  -> Task QueryObjectStoreError (QueryObjectStore query)
+newFromConfig _ = panic "not implemented: Service.QueryObjectStore.Postgres.createQueryObjectStore"

core/service/Service/Transport/Web/Readiness.hs

@@ -0,0 +1,33 @@
+module Service.Transport.Web.Readiness (
+  ReadinessConfig (..),
+  handleReadinessRequest,
+  handleQueryReadinessRequest,
+) where
+
+import Basics
+import Task (Task)
+import Text (Text)
+
+
+-- | Configuration for the /ready endpoint.
+data ReadinessConfig = ReadinessConfig
+  { readinessPath :: Text
+    -- ^ URL path for the readiness endpoint (default: "ready").
+  , includeQueryStatus :: Bool
+    -- ^ Whether /ready includes per-query lag and names (default: True).
+  }
+  deriving (Eq, Show)
+
+
+-- | Handle GET /ready.
+--
+-- Stub — not implemented.
+handleReadinessRequest :: Task Text Unit
+handleReadinessRequest = panic "not implemented: Service.Transport.Web.Readiness.handleReadinessRequest"
+
+
+-- | Handle GET /queries/{name} readiness degradation.
+--
+-- Stub — not implemented.
+handleQueryReadinessRequest :: Task Text Unit
+handleQueryReadinessRequest = panic "not implemented: Service.Transport.Web.Readiness.handleQueryReadinessRequest"

core/test-service/Main.hs

@@ -44,6 +44,10 @@ import Service.Command.AuthSpec qualified
 import Service.Command.CanAccessSpec qualified
 import Service.Transport.Web.CommandAuthSpec qualified
 import Service.CommandExecutor.AuditLoggingSpec qualified
+import Service.QueryObjectStore.PostgresSpec qualified
+import Service.Query.Subscriber.ReadinessSpec qualified
+import Service.Transport.Web.ReadinessSpec qualified
+import Service.Application.ReadinessBuilderSpec qualified
 import Test.Hspec qualified as Hspec
 
 
@@ -91,3 +95,7 @@ main = Hspec.hspec do
   Hspec.describe "Service.Command.CanAccess" Service.Command.CanAccessSpec.spec
   Hspec.describe "Service.Transport.Web.CommandAuth" Service.Transport.Web.CommandAuthSpec.spec
   Hspec.describe "Service.CommandExecutor.AuditLogging" Service.CommandExecutor.AuditLoggingSpec.spec
+  Hspec.describe "Service.QueryObjectStore.Postgres" Service.QueryObjectStore.PostgresSpec.spec
+  Hspec.describe "Service.Query.Subscriber.Readiness" Service.Query.Subscriber.ReadinessSpec.spec
+  Hspec.describe "Service.Transport.Web.Readiness" Service.Transport.Web.ReadinessSpec.spec
+  Hspec.describe "Service.Application.ReadinessBuilder" Service.Application.ReadinessBuilderSpec.spec

core/test-service/Service/Application/ReadinessBuilderSpec.hs

@@ -0,0 +1,77 @@
+module Service.Application.ReadinessBuilderSpec where
+
+import Core
+import Service.Application qualified as Application
+import Service.QueryObjectStore.Postgres (
+  PostgresQueryObjectStoreConfig (..),
+  )
+import Test
+
+
+-- | Minimal Postgres config for builder tests.
+testPostgresConfig :: PostgresQueryObjectStoreConfig
+testPostgresConfig =
+  PostgresQueryObjectStoreConfig
+    { host = "localhost"
+    , databaseName = "neohaskell"
+    , user = "neohaskell"
+    , password = "neohaskell"
+    , port = 5432
+    }
+
+
+spec :: Spec Unit
+spec = do
+  describe "useQueryObjectStore" do
+    it "wires the QueryObjectStore into the subscriber and returns updated Application" \_ -> do
+      let _app =
+            Application.new
+              |> Application.useQueryObjectStore testPostgresConfig
+      -- Stub: useQueryObjectStore throws error "not implemented", so this panics.
+      -- The test must fail, not pass.
+      fail "useQueryObjectStore wiring: not implemented — stub must fail"
+
+    it "allows chaining with other builder methods (withQuery, withEventStore, etc.)" \_ -> do
+      -- Builder chain:
+      --   Application.new |> useQueryObjectStore config |> ...
+      -- Stub panics before the chain completes.
+      fail "useQueryObjectStore chaining: not implemented — stub must fail"
+
+    it "fails at runtime if the QueryObjectStore config is invalid" \_ -> do
+      let badConfig = testPostgresConfig { host = "unreachable.invalid" }
+      let _app =
+            Application.new
+              |> Application.useQueryObjectStore badConfig
+      -- Stub: not implemented; must fail.
+      fail "useQueryObjectStore invalid config: not implemented — stub must fail"
+
+    it "accepts multiple store backends if they implement QueryObjectStoreConfig" \_ -> do
+      -- Both InMemory (via existing withQueryObjectStore) and Postgres backends
+      -- must compile and be accepted by the builder.
+      fail "useQueryObjectStore multiple backends: not implemented — stub must fail"
+
+  describe "useReadinessEndpoint" do
+    it "enables the /ready HTTP endpoint and returns updated Application" \_ -> do
+      let _app =
+            Application.new
+              |> Application.useReadinessEndpoint
+      fail "useReadinessEndpoint enables /ready: not implemented — stub must fail"
+
+    it "is enabled by default (omitting the call still activates /ready)" \_ -> do
+      -- Application.new without explicit useReadinessEndpoint should still have /ready.
+      fail "useReadinessEndpoint default on: not implemented — stub must fail"
+
+    it "allows chaining with Application.withoutReadinessEndpoint to disable /ready" \_ -> do
+      let _app =
+            Application.new
+              |> Application.useReadinessEndpoint
+              |> Application.withoutReadinessEndpoint
+      fail "useReadinessEndpoint then withoutReadinessEndpoint: not implemented — stub must fail"
+
+    it "returns Application suitable for Application.run" \_ -> do
+      let _app =
+            Application.new
+              |> Application.useReadinessEndpoint
+      -- Would call Application.run but that requires full infrastructure;
+      -- stub must fail before we get there.
+      fail "useReadinessEndpoint runnable: not implemented — stub must fail"