diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index 966cfb6b..4acf9ba4 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -20,6 +20,7 @@ env: jobs: build: strategy: + fail-fast: false matrix: include: ${{ fromJson(inputs.matrix) }} diff --git a/.gitignore b/.gitignore index fdff9a8d..78fc50c4 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,9 @@ docker/prometheus-data data/ nimbledeps logs/ +.grepai/ +*.log +CLAUDE.md +AGENTS.md +.claude +.opencode diff --git a/.gitmodules b/.gitmodules index 41f5ff81..83740187 100644 --- a/.gitmodules +++ b/.gitmodules @@ -96,12 +96,6 @@ [submodule "vendor/nimble/sqlite3_abi"] path = vendor/nimble/sqlite3_abi url = https://github.com/arnetheduck/nim-sqlite3-abi -[submodule "vendor/nimble/leveldbstatic"] - path = vendor/nimble/leveldbstatic - url = https://github.com/durability-labs/nim-leveldbstatic -[submodule "vendor/nimble/datastore"] - path = vendor/nimble/datastore - url = https://github.com/durability-labs/nim-datastore [submodule "vendor/nimble/archivistdht"] path = vendor/nimble/archivistdht url = https://github.com/durability-labs/archivist-dht @@ -159,3 +153,10 @@ [submodule "vendor/nimble/zippy"] path = vendor/nimble/zippy url = https://github.com/guzba/zippy +[submodule "vendor/nimble/nim-kvstore"] + path = vendor/nimble/nim-kvstore + url = git@github.com:durability-labs/nim-kvstore.git + url = https://github.com/durability-labs/nim-kvstore.git +[submodule "vendor/nimble/threading"] + path = vendor/nimble/threading + url = https://github.com/nim-lang/threading.git diff --git a/archivist.nim b/archivist.nim index 3f37d891..13851720 100644 --- a/archivist.nim +++ b/archivist.nim @@ -147,6 +147,7 @@ when isMainModule: chronos.poll() except Exception as exc: error "Unhandled exception in async proc, aborting", msg = exc.msg + # raise exc # uncomment for stack trace quit QuitFailure try: @@ -155,6 +156,7 @@ when isMainModule: waitFor shutdown except CatchableError as error: error "Archivist Node didn't shutdown correctly", error = error.msg + # raise exc # uncomment for stacktrace quit QuitFailure notice "Exited Archivist Node" diff --git a/archivist.nim.cfg b/archivist.nim.cfg index 9b543d0d..5ce13717 100644 --- a/archivist.nim.cfg +++ b/archivist.nim.cfg @@ -4,3 +4,5 @@ --define:"chronicles_sinks=textlines[dynamic],json[dynamic],textlines[dynamic]" # enable metrics collection --define:metrics +# bypass Nim TLSF allocator to avoid cross-thread free accumulation +--define:useMalloc diff --git a/archivist.nimble b/archivist.nimble index 252efa38..bbf17026 100644 --- a/archivist.nimble +++ b/archivist.nimble @@ -11,15 +11,8 @@ import "./vendor/nimble/deps.nims" before build: exec "nim vendor" / "nimble" / "install.nims" -task update, "Sync submodules to pinned commits (safe)": - exec "git submodule sync --recursive" - exec "git submodule update --init --recursive" - -task updateUnsafe, "Reset and clean submodules to pinned commits (destructive)": - exec "git submodule sync --recursive" - exec "git submodule foreach --recursive 'git reset --hard'" - exec "git submodule foreach --recursive 'git clean -fdx'" - exec "git submodule update --init --recursive --force" +before test: + exec "nim vendor" / "nimble" / "install.nims" task test, "Run node tests": exec "nim c -r tests" / "testNode" @@ -27,6 +20,9 @@ task test, "Run node tests": task testContracts, "Run contract tests": exec "nim c -r tests" / "testContracts" +before testIntegration: + exec "nim vendor" / "nimble" / "install.nims" + task testIntegration, "Run integration tests": exec "nim c" & " --define:release" & @@ -52,6 +48,16 @@ task format, "Format code using NPH": exec findExe("nph") & " tests/" exec findExe("nph") & " tools/" +task syncModules, "Sync submodules to pinned commits (safe)": + exec "git submodule sync --recursive" + exec "git submodule update --init --recursive" + +task syncUnsafe, "Reset and clean submodules to pinned commits (destructive)": + exec "git submodule sync --recursive" + exec "git submodule foreach --recursive 'git reset --hard'" + exec "git submodule foreach --recursive 'git clean -fdx'" + exec "git submodule update --init --recursive --force" + task addDep, "Add vendored Nim dependency (git submodule)": addDepTask(thisDir()) diff --git a/archivist/archivist.nim b/archivist/archivist.nim index 96dfb2b4..7f168e31 100644 --- a/archivist/archivist.nim +++ b/archivist/archivist.nim @@ -19,7 +19,7 @@ import pkg/confutils import pkg/confutils/defs import pkg/nitro import pkg/stew/io2 -import pkg/datastore +import pkg/kvstore import pkg/ethers except Rng import ./node @@ -168,8 +168,8 @@ proc new*( ) let - discoveryStore = Datastore( - LevelDbDatastore.new(config.dataDir / ArchivistDhtProvidersNamespace).expect( + discoveryStore = KVStore( + SQLiteKVStore.new(config.dataDir / ArchivistDhtProvidersNamespace, tp).expect( "Should create discovery datastore!" ) ) @@ -185,40 +185,41 @@ proc new*( wallet = WalletRef.new(EthPrivateKey.random()) network = BlockExcNetwork.new(switch) - repoData = + repoData: KVStore = case config.repoKind of repoFS: - Datastore( - FSDatastore.new($config.dataDir, depth = 5).expect( - "Should create repo file data store!" + KVStore( + FSKVStore + .new( + $config.dataDir, + tp, + depth = 5, + directIO = config.fsDirectIO, + fsyncFile = config.fsFsyncFile, + fsyncDir = config.fsFsyncDir, ) + .expect("Should create repo file data store!") ) of repoSQLite: - Datastore( - SQLiteDatastore.new($config.dataDir).expect( + KVStore( + SQLiteKVStore.new($config.dataDir, tp).expect( "Should create repo SQLite data store!" ) ) - of repoLevelDb: - Datastore( - LevelDbDatastore.new($config.dataDir).expect( - "Should create repo LevelDB data store!" - ) - ) repoStore = RepoStore.new( repoDs = repoData, - metaDs = LevelDbDatastore.new(config.dataDir / ArchivistMetaNamespace).expect( + metaDs = SQLiteKVStore.new(config.dataDir / ArchivistMetaNamespace, tp).expect( "Should create metadata store!" ), quotaMaxBytes = config.storageQuota, - blockTtl = config.blockTtl, + overlayTtl = config.overlayTtl.seconds, ) maintenance = BlockMaintainer.new( repoStore, - interval = config.blockMaintenanceInterval, - numberOfBlocksPerInterval = config.blockMaintenanceNumberOfBlocks, + interval = config.overlayMaintenanceInterval, + numberOfBlocksPerInterval = config.overlayMaintenanceNumberOfBlocks, ) peerStore = PeerCtxStore.new() @@ -240,6 +241,7 @@ proc new*( archivistNode = ArchivistNodeRef.new( switch = switch, networkStore = store, + repoStore = repoStore, engine = engine, discovery = discovery, prover = prover, diff --git a/archivist/blockexchange/engine/advertiser.nim b/archivist/blockexchange/engine/advertiser.nim index c31e7341..d418950f 100644 --- a/archivist/blockexchange/engine/advertiser.nim +++ b/archivist/blockexchange/engine/advertiser.nim @@ -83,12 +83,21 @@ proc advertiseBlock(b: Advertiser, cid: Cid) {.async: (raises: [CancelledError]) proc advertiseLocalStoreLoop(b: Advertiser) {.async: (raises: []).} = try: while b.advertiserRunning: - if cidsIter =? await b.localStore.listBlocks(blockType = BlockType.Manifest): - trace "Advertiser begins iterating blocks..." - for c in cidsIter: - if cid =? await c: - await b.advertiseBlock(cid) - trace "Advertiser iterating blocks finished." + without cidsIter =? await b.localStore.listBlocks(blockType = BlockType.Manifest), + err: + trace "Error retrieving manifest iterator, advertising skipped!", err = err.msg + await sleepAsync(b.advertiseLocalStoreLoopSleep) + continue + + defer: + if err =? (await cidsIter.dispose()).errorOption: + warn "Error disposing manifest iterator", err = err.msg + + trace "Advertiser begins iterating blocks..." + for c in cidsIter: + if cid =? await c: + await b.advertiseBlock(cid) + trace "Advertiser iterating blocks finished." await sleepAsync(b.advertiseLocalStoreLoopSleep) except CancelledError: @@ -126,7 +135,7 @@ proc start*(b: Advertiser) {.async: (raises: []).} = # The advertiser is expected to be started only once. if b.advertiserRunning: - raiseAssert "Advertiser can only be started once — this should not happen" + raiseAssert "Advertiser can only be started once - this should not happen" proc onBlock(cid: Cid) {.async: (raises: []).} = try: diff --git a/archivist/blockexchange/engine/engine.nim b/archivist/blockexchange/engine/engine.nim index 710a5a8b..e4890bd8 100644 --- a/archivist/blockexchange/engine/engine.nim +++ b/archivist/blockexchange/engine/engine.nim @@ -12,6 +12,7 @@ import std/sets import std/options import std/algorithm import std/sugar +import std/tables import pkg/chronos import pkg/libp2p/[cid, switch, multihash, multicodec] @@ -396,10 +397,17 @@ proc validateBlockDelivery(self: BlockExcEngine, bd: BlockDelivery): ?!void = proc blocksDeliveryHandler*( self: BlockExcEngine, peer: PeerId, blocksDelivery: seq[BlockDelivery] -) {.async: (raises: []).} = +) {.async: (raises: [CancelledError]).} = + # TODO: this should not be here, the engine should just resolve the future, + # and let the caller do the validation and storing trace "Received blocks from peer", peer, blocks = (blocksDelivery.mapIt(it.address)) - var validatedBlocksDelivery: seq[BlockDelivery] + # Validate all deliveries and separate leaf vs non-leaf + var + validatedBlocksDelivery: seq[BlockDelivery] + leafByTree: Table[Cid, seq[(Block, Natural, ArchivistProof)]] + nonLeafDeliveries: seq[BlockDelivery] + for bd in blocksDelivery: logScope: peer = peer @@ -410,36 +418,54 @@ proc blocksDeliveryHandler*( warn "Block validation failed", msg = err.msg continue - if err =? (await self.localStore.putBlock(bd.blk)).errorOption: - error "Unable to store block", err = err.msg - continue - if bd.address.leaf: without proof =? bd.proof: warn "Proof expected for a leaf block delivery" continue - if err =? ( - await self.localStore.putCidAndProof( - bd.address.treeCid, bd.address.index, bd.blk.cid, proof - ) - ).errorOption: - warn "Unable to store proof and cid for a block" - continue + + leafByTree.withValue(bd.address.treeCid, slot): + slot[].add((bd.blk, bd.address.index, proof)) + do: + leafByTree[bd.address.treeCid] = @[(bd.blk, bd.address.index, proof)] + else: + nonLeafDeliveries.add(bd) except CatchableError as exc: warn "Error handling block delivery", error = exc.msg continue validatedBlocksDelivery.add(bd) + # Batch write leaf blocks grouped by treeCid + for treeCid, items in leafByTree: + if err =? (await self.localStore.putBlocks(treeCid, items)).errorOption: + error "Unable to store leaf blocks", treeCid, err = err.msg + # Remove failed leaves from validatedBlocksDelivery + validatedBlocksDelivery.keepItIf( + not (it.address.leaf and it.address.treeCid == treeCid) + ) + + # Write non-leaf blocks sequentially - this should only be manifests after #94 + for bd in nonLeafDeliveries: + without isManifest =? bd.blk.cid.isManifest, err: + error "Received a non-leaf block that isn't a manifest!", err = err.msg + validatedBlocksDelivery.keepItIf(it.address.cid != bd.address.cid) + continue + + # TODO: The putBlock here should be replace by something like - + # storeManifestBlock(...) + if err =? (await self.localStore.putBlock(bd.blk)).errorOption: + error "Unable to store block", err = err.msg + validatedBlocksDelivery.keepItIf(it.address.cid != bd.address.cid) + archivist_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64) let peerCtx = self.peers.get(peer) if peerCtx != nil: - if err =? catch(await self.payForBlocks(peerCtx, blocksDelivery)).errorOption: + if err =? catchAsync(await self.payForBlocks(peerCtx, blocksDelivery)).errorOption: warn "Error paying for blocks", err = err.msg return - if err =? catch(await self.resolveBlocks(validatedBlocksDelivery)).errorOption: + if err =? catchAsync(await self.resolveBlocks(validatedBlocksDelivery)).errorOption: warn "Error resolving blocks", err = err.msg return @@ -470,7 +496,11 @@ proc wantListHandler*( let have = try: - await e.address in self.localStore + if e.address.leaf: + (await self.localStore.hasBlock(e.address.treeCid, e.address.index)) |? + false + else: + (await self.localStore.hasBlock(e.address.cid)) |? false except CatchableError: # TODO: should not be necessary once we have proper exception tracking on the BlockStore interface false @@ -612,13 +642,13 @@ proc taskHandler*( proc localLookup(e: WantListEntry): Future[?!BlockDelivery] {.async.} = if e.address.leaf: (await self.localStore.getBlockAndProof(e.address.treeCid, e.address.index)).map( - (blkAndProof: (Block, ArchivistProof)) => + (blkAndProof: (Natural, Block, ArchivistProof)) => BlockDelivery( - address: e.address, blk: blkAndProof[0], proof: blkAndProof[1].some + address: e.address, blk: blkAndProof[1], proof: blkAndProof[2].some ) ) else: - (await self.localStore.getBlock(e.address)).map( + (await self.localStore.getBlock(e.address.cid)).map( (blk: Block) => BlockDelivery(address: e.address, blk: blk, proof: ArchivistProof.none) ) diff --git a/archivist/blocktype.nim b/archivist/blocktype.nim index cce482d1..adbb3e2a 100644 --- a/archivist/blocktype.nim +++ b/archivist/blocktype.nim @@ -9,6 +9,7 @@ import std/tables import std/sugar +import std/hashes export tables @@ -50,6 +51,9 @@ logutils.formatIt(LogFormat.textLines, BlockAddress): logutils.formatIt(LogFormat.json, BlockAddress): %it +func hash*(blk: Block): Hash = + hash(blk.cid.data.buffer) + proc `==`*(a, b: BlockAddress): bool = a.leaf == b.leaf and ( if a.leaf: @@ -99,6 +103,22 @@ func new*( Block(cid: cid, data: @data).success +proc new*( + T: type Block, + data: sink seq[byte], + version = CIDv1, + mcodec = Sha256HashCodec, + codec = BlockCodec, +): ?!Block = + ## Sink overload -- avoids copying when caller can transfer ownership. + ## + + let + hash = ?MultiHash.digest($mcodec, data).mapFailure + cid = ?Cid.init(version, codec, hash).mapFailure + + Block(cid: cid, data: move data).success + proc new*( T: type Block, cid: Cid, data: openArray[byte], verify: bool = true ): ?!Block = @@ -115,6 +135,20 @@ proc new*( return Block(cid: cid, data: @data).success +proc new*(T: type Block, cid: Cid, data: sink seq[byte], verify: bool = true): ?!Block = + ## Sink overload -- avoids copying when caller can transfer ownership. + ## + + if verify: + let + mhash = ?cid.mhash.mapFailure + computedMhash = ?MultiHash.digest($mhash.mcodec, data).mapFailure + computedCid = ?Cid.init(cid.cidver, cid.mcodec, computedMhash).mapFailure + if computedCid != cid: + return "Cid doesn't match the data".failure + + return Block(cid: cid, data: move data).success + proc emptyBlock*(version: CidVersion, hcodec: MultiCodec): ?!Block = emptyCid(version, hcodec, BlockCodec).flatMap( (cid: Cid) => Block.new(cid = cid, data = @[]) diff --git a/archivist/chunker.nim b/archivist/chunker.nim index de15cee5..8f09d689 100644 --- a/archivist/chunker.nim +++ b/archivist/chunker.nim @@ -25,10 +25,9 @@ const DefaultChunkSize* = DefaultBlockSize type # default reader type - ChunkerError* = object of CatchableError ChunkBuffer* = ptr UncheckedArray[byte] - Reader* = proc(data: ChunkBuffer, len: int): Future[int] {. - gcsafe, async: (raises: [ChunkerError, CancelledError]) + Reader* = proc(data: ChunkBuffer, len: int): Future[?!int] {. + gcsafe, async: (raises: [CancelledError]) .} # Reader that splits input data into fixed-size chunks @@ -41,23 +40,23 @@ type FileChunker* = Chunker LPStreamChunker* = Chunker -proc getBytes*(c: Chunker): Future[seq[byte]] {.async.} = +proc getBytes*(c: Chunker): Future[?!seq[byte]] {.async: (raises: [CancelledError]).} = ## returns a chunk of bytes from ## the instantiated chunker ## var buff = newSeq[byte](c.chunkSize.int) - let read = await c.reader(cast[ChunkBuffer](addr buff[0]), buff.len) + let read = ?await c.reader(cast[ChunkBuffer](addr buff[0]), buff.len) if read <= 0: - return @[] + return success(newSeq[byte](0)) c.offset += read if not c.pad and buff.len > read: buff.setLen(read) - return move buff + success move buff proc new*( T: type Chunker, reader: Reader, chunkSize = DefaultChunkSize, pad = true @@ -74,23 +73,17 @@ proc new*( proc reader( data: ChunkBuffer, len: int - ): Future[int] {.gcsafe, async: (raises: [ChunkerError, CancelledError]).} = + ): Future[?!int] {.gcsafe, async: (raises: [CancelledError]).} = var res = 0 try: while res < len: res += await stream.readOnce(addr data[res], len - res) - except LPStreamEOFError as exc: - trace "LPStreamChunker stream Eof", exc = exc.msg - except CancelledError as error: - raise error - except LPStreamError as error: - error "LPStream error", err = error.msg - raise newException(ChunkerError, "LPStream error", error) - except CatchableError as exc: - error "CatchableError exception", exc = exc.msg - raise newException(Defect, exc.msg) - - return res + except LPStreamEOFError: + discard # EOF reached, return bytes read so far + except LPStreamError as exc: + return failure(exc) + + return success res LPStreamChunker.new(reader = reader, chunkSize = chunkSize, pad = pad) @@ -102,23 +95,15 @@ proc new*( proc reader( data: ChunkBuffer, len: int - ): Future[int] {.gcsafe, async: (raises: [ChunkerError, CancelledError]).} = + ): Future[?!int] {.gcsafe, async: (raises: [CancelledError]).} = var total = 0 - try: - while total < len: - let res = file.readBuffer(addr data[total], len - total) - if res <= 0: - break - - total += res - except IOError as exc: - trace "Exception reading file", exc = exc.msg - except CancelledError as error: - raise error - except CatchableError as exc: - error "CatchableError exception", exc = exc.msg - raise newException(Defect, exc.msg) - - return total + while total < len: + let res = ?catch(file.readBuffer(addr data[total], len - total)) + if res <= 0: + break + + total += res + + return success total FileChunker.new(reader = reader, chunkSize = chunkSize, pad = pad) diff --git a/archivist/conf.nim b/archivist/conf.nim index 34966a51..8b10bb7b 100644 --- a/archivist/conf.nim +++ b/archivist/conf.nim @@ -46,8 +46,8 @@ export units, net, archivisttypes, logutils, completeCmdArg, parseCmdArg, NatCon export ValidationGroups, MaxSlots export - DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, - DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas + DefaultQuotaBytes, DefaultOverlayTtl, DefaultBlockInterval, + DefaultNumBlocksPerInterval, DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas type ThreadCount* = range[0 .. 256] @@ -64,9 +64,6 @@ proc defaultDataDir*(): string = const DefaultDataDir* = defaultDataDir() -proc defaultCircuitDir*(): string = - defaultDataDir() / "circuits" - proc toAbsolutePath*(path: string): string = try: absolutePath(path) @@ -91,7 +88,6 @@ type RepoKind* = enum repoFS = "fs" repoSQLite = "sqlite" - repoLevelDb = "leveldb" NodeConf* = object configFile* {. @@ -220,12 +216,37 @@ type .}: Option[string] repoKind* {. - desc: "Backend for main repo store (fs, sqlite, leveldb)", + desc: "Backend for main repo store (fs, sqlite)", defaultValueDesc: "fs", defaultValue: repoFS, name: "repo-kind" .}: RepoKind + fsDirectIO* {. + desc: + "Use O_DIRECT for filesystem writes (bypass page cache). " & + "FS backend only. May cause EINVAL on some platforms.", + defaultValue: false, + defaultValueDesc: "false", + name: "fs-direct-io" + .}: bool + + fsFsyncFile* {. + desc: "Fsync files after write in filesystem backend. " & "FS backend only.", + defaultValue: true, + defaultValueDesc: "true", + name: "fs-fsync-file" + .}: bool + + fsFsyncDir* {. + desc: + "Fsync parent directory after rename/delete in filesystem backend. " & + "FS backend only.", + defaultValue: true, + defaultValueDesc: "true", + name: "fs-fsync-dir" + .}: bool + storageQuota* {. desc: "The size of the total storage quota dedicated to the node", defaultValue: DefaultQuotaBytes, @@ -234,15 +255,15 @@ type abbr: "q" .}: NBytes - blockTtl* {. - desc: "Default block timeout in seconds - 0 disables the ttl", - defaultValue: DefaultBlockTtl, - defaultValueDesc: $DefaultBlockTtl, - name: "block-ttl", + overlayTtl* {. + desc: "Default overlay timeout in seconds - 0 disables the ttl", + defaultValue: DefaultOverlayTtl.seconds, + defaultValueDesc: $DefaultOverlayTtl, + name: "overlay-ttl", abbr: "t" .}: Duration - blockMaintenanceInterval* {. + overlayMaintenanceInterval* {. desc: "Time interval in seconds - determines frequency of block " & "maintenance cycle: how often blocks are checked " & "for expiration and cleanup", @@ -251,7 +272,7 @@ type name: "block-mi" .}: Duration - blockMaintenanceNumberOfBlocks* {. + overlayMaintenanceNumberOfBlocks* {. desc: "Number of blocks to check every maintenance cycle", defaultValue: DefaultNumBlocksPerInterval, defaultValueDesc: $DefaultNumBlocksPerInterval, @@ -373,8 +394,8 @@ type circuitDir* {. desc: "Directory where the node will store proof circuit data", - defaultValue: defaultCircuitDir(), - defaultValueDesc: "data/circuits", + defaultValue: OutDir($config.dataDir / "circuits"), + defaultValueDesc: "/circuits", abbr: "cd", name: "circuit-dir" .}: OutDir diff --git a/archivist/discovery.nim b/archivist/discovery.nim index e5da4d3e..64a0a4cb 100644 --- a/archivist/discovery.nim +++ b/archivist/discovery.nim @@ -18,7 +18,7 @@ import pkg/libp2p/[cid, multicodec, routing_record, signed_envelope] import pkg/questionable import pkg/questionable/results import pkg/contractabi/address as ca -import pkg/datastore +import pkg/kvstore import pkg/archivistdht/discv5/[routing_table, protocol as discv5] from pkg/nimcrypto import keccak256 @@ -208,10 +208,11 @@ proc start*(d: Discovery) {.async: (raises: []).} = error "Error starting discovery", exc = exc.msg proc stop*(d: Discovery) {.async: (raises: []).} = - try: - await noCancel d.protocol.closeWait() - except CatchableError as exc: - error "Error stopping discovery", exc = exc.msg + if not d.protocol.isNil and not d.protocol.transport.isNil: + try: + await noCancel d.protocol.closeWait() + except CatchableError as exc: + error "Error stopping discovery", exc = exc.msg proc new*( T: type Discovery, @@ -220,7 +221,7 @@ proc new*( bindPort = 0.Port, announceAddrs: openArray[MultiAddress], bootstrapNodes: openArray[SignedPeerRecord] = [], - store: Datastore = SQLiteDatastore.new(datastore.Memory).expect("Should not fail!"), + store: KVStore, ): Discovery = ## Create a new Discovery node instance for the given key and datastore ## diff --git a/archivist/erasure/erasure.nim b/archivist/erasure/erasure.nim index cd453893..26139d7a 100644 --- a/archivist/erasure/erasure.nim +++ b/archivist/erasure/erasure.nim @@ -70,15 +70,19 @@ type taskPool: Taskpool encoderProvider*: EncoderProvider decoderProvider*: DecoderProvider - store*: BlockStore + networkStore*: BlockStore + repoStore*: RepoStore EncodingParams = object + blockSize: NBytes ecK: Natural ecM: Natural rounded: Natural steps: Natural blocksCount: Natural + encodedBlocksCount: Natural strategy: StrategyType + emptyCid: Cid ErasureError* = object of ArchivistError InsufficientBlocksError* = object of ErasureError @@ -115,7 +119,7 @@ func indexToPos(steps, idx, step: int): int {.inline.} = (idx - step) div steps proc getPendingBlocks( - self: Erasure, manifest: Manifest, indices: seq[int] + self: Erasure, treeCid: Cid, indices: seq[int] ): AsyncIter[(?!bt.Block, int)] = ## Get pending blocks iterator ## @@ -129,7 +133,7 @@ proc getPendingBlocks( for blockIndex in indices: # request blocks from the store - let fut = self.store.getBlock(BlockAddress.init(manifest.treeCid, blockIndex)) + let fut = self.networkStore.getBlock(treeCid, blockIndex) pendingBlocks.add(attachIndex(fut, blockIndex)) proc isFinished(): bool = @@ -144,38 +148,48 @@ proc getPendingBlocks( let (_, index) = await completedFut raise newException( CatchableError, - "Future for block id not found, tree cid: " & $manifest.treeCid & ", index: " & - $index, + "Future for block id not found, tree cid: " & $treeCid & ", index: " & $index, ) AsyncIter[(?!bt.Block, int)].new(genNext, isFinished) proc prepareEncodingData( self: Erasure, - manifest: Manifest, + treeCid: Cid, params: EncodingParams, step: Natural, data: ref seq[seq[byte]], cids: ref seq[Cid], + emptyCid: Cid, emptyBlock: seq[byte], -): Future[?!Natural] {.async.} = +): Future[?!Natural] {.async: (raises: [CancelledError]).} = ## Prepare data for encoding ## + logScope: + treeCid = treeCid + step = step + blocksCount = params.blocksCount + rounded = params.rounded + strategy = params.strategy + + trace "Preparing encoding data" let - strategy = params.strategy.init( - firstIndex = 0, lastIndex = params.rounded - 1, iterations = params.steps - ) - indices = toSeq(strategy.getIndices(step)) + strategy = + ?catch( + params.strategy.init( + firstIndex = 0, lastIndex = params.rounded - 1, iterations = params.steps + ) + ) + indices = toSeq(?catch(strategy.getIndices(step))) pendingBlocksIter = - self.getPendingBlocks(manifest, indices.filterIt(it < manifest.blocksCount)) + self.getPendingBlocks(treeCid, indices.filterIt(it < params.blocksCount)) var resolved = 0 for fut in pendingBlocksIter: - let (blkOrErr, idx) = await fut - without blk =? blkOrErr, err: - warn "Failed retrieving a block", treeCid = manifest.treeCid, idx, msg = err.msg - return failure(err) + let + (blkOrErr, idx) = ?catchAsync(await fut) + blk = ?blkOrErr let pos = indexToPos(params.steps, idx, step) data[pos] = if blk.isEmpty: emptyBlock else: blk.data @@ -183,28 +197,27 @@ proc prepareEncodingData( resolved.inc() - for idx in indices.filterIt(it >= manifest.blocksCount): + for idx in indices.filterIt(it >= params.blocksCount): let pos = indexToPos(params.steps, idx, step) trace "Padding with empty block", idx data[pos] = emptyBlock - without emptyBlockCid =? emptyCid(manifest.version, manifest.hcodec, manifest.codec), - err: - return failure(err) - cids[idx] = emptyBlockCid + cids[idx] = emptyCid success(resolved.Natural) proc prepareDecodingData( self: Erasure, - encoded: Manifest, + treeCid: Cid, + params: EncodingParams, step: Natural, data: ref seq[seq[byte]], parityData: ref seq[seq[byte]], cids: ref seq[Cid], emptyBlock: seq[byte], -): Future[?!(Natural, Natural)] {.async.} = +): Future[?!(Natural, Natural)] {.async: (raises: [CancelledError]).} = ## Prepare data for decoding - ## `encoded` - the encoded manifest + ## `treeCid` - the tree cid to fetch blocks from + ## `params` - encoding parameters ## `step` - the current step ## `data` - the data to be prepared ## `parityData` - the parityData to be prepared @@ -213,11 +226,16 @@ proc prepareDecodingData( ## let - strategy = encoded.protectedStrategy.init( - firstIndex = 0, lastIndex = encoded.blocksCount - 1, iterations = encoded.steps - ) - indices = toSeq(strategy.getIndices(step)) - pendingBlocksIter = self.getPendingBlocks(encoded, indices) + strategy = + ?catch( + params.strategy.init( + firstIndex = 0, + lastIndex = params.encodedBlocksCount - 1, + iterations = params.steps, + ) + ) + indices = toSeq(?catch(strategy.getIndices(step))) + pendingBlocksIter = self.getPendingBlocks(treeCid, indices) var dataPieces = 0 @@ -226,15 +244,15 @@ proc prepareDecodingData( for fut in pendingBlocksIter: # Continue to receive blocks until we have just enough for decoding # or no more blocks can arrive - if resolved >= encoded.ecK: + if resolved >= params.ecK: break - let (blkOrErr, idx) = await fut + let (blkOrErr, idx) = ?catchAsync(await fut) without blk =? blkOrErr, err: - trace "Failed retrieving a block", idx, treeCid = encoded.treeCid, msg = err.msg + trace "Failed retrieving a block", idx, treeCid = treeCid, msg = err.msg continue - let pos = indexToPos(encoded.steps, idx, step) + let pos = indexToPos(params.steps, idx, step) logScope: cid = blk.cid @@ -244,9 +262,9 @@ proc prepareDecodingData( empty = blk.isEmpty cids[idx] = blk.cid - if idx >= encoded.rounded: + if idx >= params.rounded: trace "Retrieved parity block" - parityData[pos - encoded.ecK] = if blk.isEmpty: emptyBlock else: blk.data + parityData[pos - params.ecK] = if blk.isEmpty: emptyBlock else: blk.data parityPieces.inc else: trace "Retrieved data block" @@ -263,6 +281,7 @@ proc init*( ecK: Natural, ecM: Natural, strategy: StrategyType, + emptyCid: Cid, ): ?!EncodingParams = if ecK > manifest.blocksCount: let exc = (ref InsufficientBlocksError)( @@ -276,19 +295,37 @@ proc init*( let rounded = roundUp(manifest.blocksCount, ecK) steps = divUp(rounded, ecK) - blocksCount = rounded + (steps * ecM) success EncodingParams( + blockSize: manifest.blockSize, ecK: ecK, ecM: ecM, rounded: rounded, steps: steps, - blocksCount: blocksCount, + blocksCount: manifest.blocksCount, + encodedBlocksCount: rounded + (steps * ecM), strategy: strategy, + emptyCid: emptyCid, + ) + +proc initFromEncoded*( + _: type EncodingParams, encoded: Manifest, emptyCid: Cid +): EncodingParams = + ## Construct EncodingParams from an already-encoded manifest. + ## + EncodingParams( + blockSize: encoded.blockSize, + ecK: encoded.ecK, + ecM: encoded.ecM, + rounded: encoded.rounded, + steps: encoded.steps, + blocksCount: encoded.originalBlocksCount, + encodedBlocksCount: encoded.blocksCount, + strategy: encoded.protectedStrategy, + emptyCid: emptyCid, ) proc leopardEncodeTask(tp: Taskpool, task: ptr EncodeTask) {.gcsafe.} = - # Task suitable for running in taskpools - look, no GC! let encoder = task[].erasure.encoderProvider( task[].blockSize, task[].blocks[].len, task[].parity[].len ) @@ -343,94 +380,61 @@ proc asyncEncode*( success() proc encodeData( - self: Erasure, manifest: Manifest, params: EncodingParams -): Future[?!Manifest] {.async: (raises: [CancelledError]).} = - ## Encode blocks pointed to by the protected manifest - ## - ## `manifest` - the manifest to encode + self: Erasure, originalTreeCid: Cid, tmpTreeCid: Cid, params: EncodingParams +): Future[?!ref seq[Cid]] {.async: (raises: [CancelledError]).} = + ## Encode blocks and return the cids of all blocks (data + parity). + ## Tree construction, proof storage, and manifest creation are + ## handled by the caller. ## logScope: + originalTreeCid = originalTreeCid + tmpTreeCid = tmpTreeCid steps = params.steps - rounded_blocks = params.rounded - blocks_count = params.blocksCount + roundedBlocks = params.rounded + blocksCount = params.blocksCount + encodedBlocksCount = params.encodedBlocksCount ecK = params.ecK ecM = params.ecM + trace "Starting erasure coding dataset" var cids = seq[Cid].new() - emptyBlock = newSeq[byte](manifest.blockSize.int) + emptyBlock = newSeq[byte](params.blockSize.int) - cids[].setLen(params.blocksCount) + cids[].setLen(params.encodedBlocksCount) + for step in 0 ..< params.steps: + # TODO: Don't allocate a new seq every time, allocate once and zero out + var + data = new seq[seq[byte]] # number of blocks to encode + parity = new seq[seq[byte]] - try: - for step in 0 ..< params.steps: - # TODO: Don't allocate a new seq every time, allocate once and zero out - var - data = new seq[seq[byte]] # number of blocks to encode - parity = new seq[seq[byte]] + data[].setLen(params.ecK) + parity[] = newSeqWith(params.ecM, newSeqWith(params.blockSize.int, 0'u8)) - data[].setLen(params.ecK) - parity[] = newSeqWith(params.ecM, newSeqWith(manifest.blockSize.int, 0'u8)) - - # TODO: this is a tight blocking loop so we sleep here to allow - # other events to be processed, this should be addressed - # by threading - await sleepAsync(10.millis) - - without resolved =? - (await self.prepareEncodingData(manifest, params, step, data, cids, emptyBlock)), - err: - trace "Unable to prepare data", error = err.msg - return failure(err) - - trace "Erasure coding data", data = data[].len + let resolved = + ?await self.prepareEncodingData( + originalTreeCid, params, step, data, cids, params.emptyCid, emptyBlock + ) - try: - if err =? - (await self.asyncEncode(manifest.blockSize.int, data, parity)).errorOption: - return failure(err) - except CancelledError as exc: - raise exc + trace "Erasure coding data", data = data[].len + ?await self.asyncEncode(params.blockSize.int, data, parity) + var + idx = params.rounded + step + blocks: seq[(bt.Block, Natural, ArchivistProof)] - var idx = params.rounded + step - for j in 0 ..< params.ecM: - without blk =? bt.Block.new(parity[j]), error: - trace "Unable to create parity block", err = error.msg - return failure(error) + for j in 0 ..< params.ecM: + let blk = ?bt.Block.new(parity[j]) - trace "Adding parity block", cid = blk.cid, idx - cids[idx] = blk.cid - if error =? (await self.store.putBlock(blk)).errorOption: - warn "Unable to store block!", cid = blk.cid, msg = error.msg - return failure("Unable to store block!") - idx.inc(params.steps) + trace "Adding parity block", cid = blk.cid, idx + cids[idx] = blk.cid + blocks.add((blk, idx.Natural, nil)) + idx.inc(params.steps) - without tree =? ArchivistTree.init(cids[]), err: - return failure(err) + trace "Storing parity blocks", count = blocks.len + ?await self.repoStore.putBlocks(tmpTreeCid, blocks) - without treeCid =? tree.rootCid, err: - return failure(err) - - if err =? (await self.store.putAllProofs(tree)).errorOption: - return failure(err) - - let encodedManifest = Manifest.new( - manifest = manifest, - treeCid = treeCid, - datasetSize = (manifest.blockSize.int * params.blocksCount).NBytes, - ecK = params.ecK, - ecM = params.ecM, - strategy = params.strategy, - ) - - trace "Encoded data successfully", treeCid, blocksCount = params.blocksCount - success encodedManifest - except CancelledError as exc: - trace "Erasure coding encoding cancelled" - raise exc # cancellation needs to be propagated - except CatchableError as exc: - trace "Erasure coding encoding error", exc = exc.msg - return failure(exc) + trace "Encoding complete", encodedBlocksCount = params.encodedBlocksCount + success cids proc encode*( self: Erasure, @@ -446,13 +450,64 @@ proc encode*( ## `parity` - the number of parity blocks to generate - M ## - without params =? EncodingParams.init(manifest, blocks.int, parity.int, strategy), err: - return failure(err) + let + emptyCid = ?emptyCid(manifest.version, manifest.hcodec, manifest.codec) + sourceManifest = + if manifest.protected: + Manifest.new(manifest) + else: + manifest + params = + ?EncodingParams.init(sourceManifest, blocks.int, parity.int, strategy, emptyCid) + + proc treeAndProofs( + cids: ref seq[Cid], targetCid: Cid + ): Future[?!Cid] {.async: (raises: [CancelledError]).} = + let + tree = ?ArchivistTree.init(cids[]) + treeCid = ?tree.rootCid + + var proofItems: seq[(Natural, Cid, ArchivistProof)] + for index, cid in cids[]: + proofItems.add((index.Natural, cid, ?tree.getProof(index))) + + ?await self.repoStore.putCidsAndProofs(targetCid, proofItems) + + trace "Tree and proofs stored", treeCid, blocks = blocks, parity = parity + success treeCid + + let treeCid = + if manifest.protected: + ?await self.repoStore.withOverlay( + manifest.treeCid, + body = proc(): Future[?!Cid] {. + closure, gcsafe, async: (raises: [CancelledError]) + .} = + let cids = + ?await self.encodeData(sourceManifest.treeCid, manifest.treeCid, params) + await treeAndProofs(cids, manifest.treeCid) + , + ) + else: + ?await self.repoStore.withTmpOverlay( + body = proc( + tmpCid: Cid + ): Future[?!Cid] {.closure, gcsafe, async: (raises: [CancelledError]).} = + let cids = ?await self.encodeData(sourceManifest.treeCid, tmpCid, params) - without encodedManifest =? await self.encodeData(manifest, params), err: - return failure(err) + await treeAndProofs(cids, tmpCid) + ) - return success encodedManifest + let protected = Manifest.new( + manifest = sourceManifest, + treeCid = treeCid, + ecK = blocks.int, + ecM = parity.int, + strategy = strategy, + datasetSize = (sourceManifest.blockSize.int * params.encodedBlocksCount).NBytes, + ) + + success(protected) proc leopardDecodeTask(tp: Taskpool, task: ptr DecodeTask) {.gcsafe.} = # Task suitable for running in taskpools - look, no GC! @@ -513,81 +568,59 @@ proc asyncDecode*( success() proc decodeInternal( - self: Erasure, encoded: Manifest + self: Erasure, encodedTreeCid: Cid, targetCid: Cid, params: EncodingParams ): Future[?!(ref seq[Cid], seq[Natural])] {.async: (raises: [CancelledError]).} = logScope: - steps = encoded.steps - rounded_blocks = encoded.rounded - new_manifest = encoded.blocksCount + encodedTreeCid = encodedTreeCid + targetCid = targetCid + steps = params.steps + roundedBlocks = params.rounded + encodedBlocksCount = params.encodedBlocksCount var cids = seq[Cid].new() recoveredIndices = newSeq[Natural]() - decoder = self.decoderProvider(encoded.blockSize.int, encoded.ecK, encoded.ecM) - emptyBlock = newSeq[byte](encoded.blockSize.int) - - cids[].setLen(encoded.blocksCount) - try: - for step in 0 ..< encoded.steps: - # TODO: this is a tight blocking loop so we sleep here to allow - # other events to be processed, this should be addressed - # by threading - await sleepAsync(10.millis) - - var - data = new seq[seq[byte]] - parityData = new seq[seq[byte]] - recovered = new seq[seq[byte]] - - data[].setLen(encoded.ecK) # set len to K - parityData[].setLen(encoded.ecM) # set len to M - recovered[] = newSeqWith(encoded.ecK, newSeqWith(encoded.blockSize.int, 0'u8)) - - without (dataPieces, _) =? ( - await self.prepareDecodingData( - encoded, step, data, parityData, cids, emptyBlock - ) - ), err: - trace "Unable to prepare data", error = err.msg - return failure(err) + emptyBlock = newSeq[byte](params.blockSize.int) + + cids[].setLen(params.encodedBlocksCount) + for step in 0 ..< params.steps: + var + data = new seq[seq[byte]] + parityData = new seq[seq[byte]] + recovered = new seq[seq[byte]] + + data[].setLen(params.ecK) # set len to K + parityData[].setLen(params.ecM) # set len to M + recovered[] = newSeqWith(params.ecK, newSeqWith(params.blockSize.int, 0'u8)) + + let (dataPieces, _) = + ?await self.prepareDecodingData( + encodedTreeCid, params, step, data, parityData, cids, emptyBlock + ) - if dataPieces >= encoded.ecK: - trace "Retrieved all the required data blocks" - continue - - trace "Erasure decoding data" - try: - if err =? ( - await self.asyncDecode(encoded.blockSize.int, data, parityData, recovered) - ).errorOption: - return failure(err) - except CancelledError as exc: - raise exc - - for i in 0 ..< encoded.ecK: - let idx = i * encoded.steps + step - if data[i].len <= 0 and not cids[idx].isEmpty: - without blk =? bt.Block.new(recovered[i]), error: - trace "Unable to create block!", exc = error.msg - return failure(error) - - trace "Recovered block", cid = blk.cid, index = i - if error =? (await self.store.putBlock(blk)).errorOption: - warn "Unable to store block!", cid = blk.cid, msg = error.msg - return failure("Unable to store block!") - - self.store.completeBlock(BlockAddress.init(encoded.treeCid, idx), blk) - - cids[idx] = blk.cid - recoveredIndices.add(idx) - except CancelledError as exc: - trace "Erasure coding decoding cancelled" - raise exc # cancellation needs to be propagated - except CatchableError as exc: - trace "Erasure coding decoding error", exc = exc.msg - return failure(exc) - finally: - decoder.release() + if dataPieces >= params.ecK: + trace "Retrieved all the required data blocks" + continue + + trace "Erasure decoding data" + ?await self.asyncDecode(params.blockSize.int, data, parityData, recovered) + var blocks: seq[(bt.Block, Natural, ArchivistProof)] + for i in 0 ..< params.ecK: + let idx = i * params.steps + step + if data[i].len <= 0 and not cids[idx].isEmpty: + without blk =? bt.Block.new(recovered[i]), error: + trace "Unable to create block!", exc = error.msg + return failure(error) + + trace "Recovered block", cid = blk.cid, index = i + self.networkStore.completeBlock(encodedTreeCid, idx, blk) + + cids[idx] = blk.cid + blocks.add((blk, idx.Natural, nil)) + recoveredIndices.add(idx) + + trace "Storing recovered blocks", count = blocks.len + ?await self.repoStore.putBlocks(targetCid, blocks) return (cids, recoveredIndices).success @@ -601,60 +634,90 @@ proc decode*( ## be recovered ## - without (cids, recoveredIndices) =? (await self.decodeInternal(encoded)), err: - return failure(err) - - without tree =? ArchivistTree.init(cids[0 ..< encoded.originalBlocksCount]), err: - return failure(err) + let + emptyCid = ?emptyCid(encoded.version, encoded.hcodec, encoded.codec) + params = EncodingParams.initFromEncoded(encoded, emptyCid) - without treeCid =? tree.rootCid, err: - return failure(err) + var + cids: ref seq[Cid] + recoveredIndices: seq[Natural] + + ?await self.repoStore.withOverlay( + encoded.originalTreeCid, + status = Storing.some, + body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = + let + (cids, recoveredIndices) = + ?await self.decodeInternal(encoded.treeCid, encoded.originalTreeCid, params) + tree = ?ArchivistTree.init(cids[0 ..< encoded.originalBlocksCount]) + treeCid = ?tree.rootCid + + if treeCid != encoded.originalTreeCid: + return failure( + "Original tree root differs from the tree root computed out of recovered data" + ) - if treeCid != encoded.originalTreeCid: - return failure( - "Original tree root differs from the tree root computed out of recovered data" - ) + let idxIter = + Iter[Natural].new(recoveredIndices).filter((i: Natural) => i < tree.leavesCount) - let idxIter = - Iter[Natural].new(recoveredIndices).filter((i: Natural) => i < tree.leavesCount) + if err =? (await self.repoStore.putSomeProofs(tree, idxIter)).errorOption: + return failure(err) - if err =? (await self.store.putSomeProofs(tree, idxIter)).errorOption: - return failure(err) + success(), + ) let decoded = Manifest.new(encoded) - return decoded.success -proc repair*(self: Erasure, encoded: Manifest): Future[?!void] {.async.} = +proc repair*( + self: Erasure, encoded: Manifest +): Future[?!void] {.async: (raises: [CancelledError]).} = ## Repair a protected manifest by reconstructing the full dataset ## ## `encoded` - the encoded (protected) manifest to ## be repaired ## + ## TODO: there are several quirks to be aware of here - + ## decode can only recreate original blocks, the resulting + ## parity is not usable (AFAIK) as the original parity blocks + ## so we need decode the original dataset first and then + ## re-encode it to get valid parity blocks for the repaired manifest + ## + ## So the first decode is called with the original tree as it's overlay, + ## the call to encode will create the protected overlay and manifest, + ## this means that we need to properly handle cleanup of these overlays + ## to avoid leaving garbage in the store + ## - without (cids, _) =? (await self.decodeInternal(encoded)), err: - return failure(err) - - without tree =? ArchivistTree.init(cids[0 ..< encoded.originalBlocksCount]), err: - return failure(err) - - without treeCid =? tree.rootCid, err: - return failure(err) - - if treeCid != encoded.originalTreeCid: - return failure( - "Original tree root differs from the tree root computed out of recovered data" - ) + let + emptyCid = ?emptyCid(encoded.version, encoded.hcodec, encoded.codec) + params = EncodingParams.initFromEncoded(encoded, emptyCid) + + ?await self.repoStore.withOverlay( + encoded.originalTreeCid, + status = Repairing.some, + body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = + let + (cids, _) = + ?await self.decodeInternal(encoded.treeCid, encoded.originalTreeCid, params) + tree = ?ArchivistTree.init(cids[0 ..< encoded.originalBlocksCount]) + treeCid = ?tree.rootCid + + if treeCid != encoded.originalTreeCid: + return failure( + "Original tree root differs from the tree root computed out of recovered data" + ) - if err =? (await self.store.putAllProofs(tree)).errorOption: - return failure(err) + await self.repoStore.putAllProofs(tree) + , + ) - without repaired =? ( - await self.encode( - Manifest.new(encoded), encoded.ecK, encoded.ecM, encoded.protectedStrategy - ) - ), err: - return failure(err) + # TODO: We don't get valid parity data from leopard, + # so we need to do full re-encoding to get parity + # blocks for a valid slot - this is higly inneficient + # we need to either fix leopard or use another implementation + let repaired = + ?(await self.encode(encoded, encoded.ecK, encoded.ecM, encoded.protectedStrategy)) if repaired.treeCid != encoded.treeCid: return failure( @@ -671,7 +734,8 @@ proc stop*(self: Erasure) {.async.} = proc new*( T: type Erasure, - store: BlockStore, + networkStore: BlockStore, + repoStore: RepoStore, encoderProvider: EncoderProvider, decoderProvider: DecoderProvider, taskPool: Taskpool, @@ -679,7 +743,8 @@ proc new*( ## Create a new Erasure instance for encoding and decoding manifests ## Erasure( - store: store, + networkStore: networkStore, + repoStore: repoStore, encoderProvider: encoderProvider, decoderProvider: decoderProvider, taskPool: taskPool, diff --git a/archivist/errors.nim b/archivist/errors.nim index 5fdbcf27..cce07a7c 100644 --- a/archivist/errors.nim +++ b/archivist/errors.nim @@ -10,7 +10,6 @@ {.push raises: [].} import std/options -import std/sugar import std/sequtils import pkg/results @@ -52,6 +51,7 @@ proc allFinishedFailed*[T]( ## ## TODO: wip, not sure if we want this - at the minimum, ## we should probably avoid the async transform + ## var res: FinishedFailed[T] = (@[], @[]) await allFutures(futs) @@ -80,8 +80,25 @@ proc allFinishedValues*[T]( # here, we know there are no failed futures in "futs" # and we are only interested in those that completed successfully - let values = collect: - for b in futs: - if b.finished: - b.value - return success values + return success futs.filterIt(it.finished).mapIt(it.value) + +template catchAsync*(body: typed): Result[type(body), ref CatchableError] = + ## Catch exceptions for body and store them in the Result + ## + ## NOTE: Adopted from Results to propagate async cancellations + ## + ## ``` + ## let r = catch: someFuncThatMayRaise() + ## ``` + type R = Result[type(body), ref CatchableError] + + try: + when type(body) is void: + body + R.ok() + else: + R.ok(body) + except CancelledError as exc: + raise exc + except CatchableError as exc: + R.err(exc) diff --git a/archivist/manifest/coders.nim b/archivist/manifest/coders.nim index d7988b57..0c2e7813 100644 --- a/archivist/manifest/coders.nim +++ b/archivist/manifest/coders.nim @@ -255,3 +255,9 @@ func decode*(_: type Manifest, blk: Block): ?!Manifest = return failure "Cid not a manifest codec" Manifest.decode(blk.data) + +func toBlock*(manifest: Manifest): ?!Block = + Block.new(data = ?manifest.encode(), codec = ManifestCodec) + +func toCid*(manifest: Manifest): ?!Cid = + success (?manifest.toBlock).cid diff --git a/archivist/marketplace/availability/store.nim b/archivist/marketplace/availability/store.nim index 6c36fb25..3c5b5b11 100644 --- a/archivist/marketplace/availability/store.nim +++ b/archivist/marketplace/availability/store.nim @@ -1,24 +1,37 @@ import pkg/chronos import pkg/questionable/results -import pkg/datastore/typedds +import pkg/kvstore import ../../stores/keyutils import ./terms import ./encoding type AvailabilityStore* = ref object - store: TypedDatastore + store: KVStore const DatastoreKey = !(!(ArchivistMetaKey / "sales") / "availability") -func new*(_: type AvailabilityStore, store: TypedDatastore): AvailabilityStore = +func new*(_: type AvailabilityStore, store: KVStore): AvailabilityStore = AvailabilityStore(store: store) proc load*( availability: AvailabilityStore ): Future[?!AvailabilityTerms] {.async: (raises: [CancelledError]).} = - await get[AvailabilityTerms](availability.store, DataStoreKey) + let record = ?await get(availability.store, DatastoreKey, AvailabilityTerms) + success(record.val) proc save*( availability: AvailabilityStore, terms: AvailabilityTerms ): Future[?!void] {.async: (raises: [CancelledError]).} = - await availability.store.put(DataStoreKey, terms) + ?await availability.store.tryPut( + KVRecord[AvailabilityTerms].init(DatastoreKey, terms), + maxRetries = 3, + proc( + failed: seq[KVRecord[AvailabilityTerms]] + ): Future[?!seq[KVRecord[AvailabilityTerms]]] {.async: (raises: [CancelledError]).} = + var record = ?await availability.store.get(failed[0].key, AvailabilityTerms) + record.val = terms + success @[record] + , + ) + + success() diff --git a/archivist/marketplace/contracts/clock.nim b/archivist/marketplace/contracts/clock.nim index c1f1fc64..827d8bd7 100644 --- a/archivist/marketplace/contracts/clock.nim +++ b/archivist/marketplace/contracts/clock.nim @@ -61,5 +61,7 @@ method now*(clock: OnChainClock): SecondsSince1970 = method waitUntil*(clock: OnChainClock, time: SecondsSince1970) {.async.} = while (let difference = time - clock.now(); difference > 0): + trace "clock waitUntil sleeping", + targetTime = time, currentTime = clock.now(), difference clock.newBlock.clear() discard await clock.newBlock.wait().withTimeout(chronos.seconds(difference)) diff --git a/archivist/marketplace/node.nim b/archivist/marketplace/node.nim index 965320b6..3abc7560 100644 --- a/archivist/marketplace/node.nim +++ b/archivist/marketplace/node.nim @@ -1,7 +1,7 @@ import pkg/ethers import pkg/chronos import pkg/questionable/results -import pkg/datastore/typedds +import pkg/kvstore import ./purchasing import ./sales import ./validation @@ -30,7 +30,7 @@ proc connect*( ethProviderUrl: string, ethPrivateKeyFile: string, storage: StorageInterface, - datastore: TypedDatastore, + datastore: KVStore, options = MarketplaceOptions(), ): Future[?!MarketplaceNode] {.async: (raises: [CancelledError]).} = let provider = ?await Provider.connect(ethProviderUrl, options) diff --git a/archivist/marketplace/sales.nim b/archivist/marketplace/sales.nim index ec4f50f3..e691dcfd 100644 --- a/archivist/marketplace/sales.nim +++ b/archivist/marketplace/sales.nim @@ -2,7 +2,6 @@ import std/sequtils import pkg/questionable import pkg/questionable/results import pkg/stint -import pkg/datastore import ../clock import ../stores import ../logutils diff --git a/archivist/marketplace/sales/states/errored.nim b/archivist/marketplace/sales/states/errored.nim index 0d82c419..daf29474 100644 --- a/archivist/marketplace/sales/states/errored.nim +++ b/archivist/marketplace/sales/states/errored.nim @@ -44,11 +44,7 @@ method run*( let agent = SalesAgent(machine) let data = agent.data let marketplace = agent.context.marketplace - - error "Sale error", - error = state.error.msgDetail, - requestId = data.requestId, - slotIndex = data.slotIndex + let slotIndex = data.slotIndex try: await data.errorBackoff.applyDelay() diff --git a/archivist/marketplace/sales/states/failed.nim b/archivist/marketplace/sales/states/failed.nim index fcd513f7..b7bc5f8d 100644 --- a/archivist/marketplace/sales/states/failed.nim +++ b/archivist/marketplace/sales/states/failed.nim @@ -2,6 +2,7 @@ import pkg/chronos import ../../../logutils import ../../../utils/exceptions import ../../../marketplace/abstractmarketplace +import ../../storageinterface import ../salesagent import ../statemachine import ./types @@ -19,8 +20,11 @@ method `$`*(state: SaleFailed): string = method run*( state: SaleFailed, machine: Machine ): Future[?State] {.async: (raises: []).} = - let data = SalesAgent(machine).data - let marketplace = SalesAgent(machine).context.marketplace + let agent = SalesAgent(machine) + let data = agent.data + let context = agent.context + let marketplace = context.marketplace + let storage = context.storage without request =? data.request: raiseAssert "no sale request" @@ -32,6 +36,12 @@ method run*( await marketplace.freeSlot(slot.id) + # Delete slot from the repostore + if request =? data.request: + if err =? + (await storage.deleteSlot(request.content.cid, data.slotIndex)).errorOption: + error "Failed to mark slot as failed", error = err.msg + let error = newException(SaleFailedError, "Sale failed") return some State(SaleErrored(error: error)) except CancelledError as e: diff --git a/archivist/marketplace/storageinterface.nim b/archivist/marketplace/storageinterface.nim index e1f8fd09..2805afcf 100644 --- a/archivist/marketplace/storageinterface.nim +++ b/archivist/marketplace/storageinterface.nim @@ -34,3 +34,8 @@ method updateSlotExpiry*( storage: StorageInterface, cid: Cid, slotIndex: uint64, expiry: StorageTimestamp ): Future[?!void] {.base, async: (raises: [CancelledError]).} = raiseAssert "not implemented" + +method deleteSlot*( + storage: StorageInterface, cid: Cid, slotIndex: uint64 +): Future[?!void] {.base, async: (raises: [CancelledError]).} = + raiseAssert "not implemented" diff --git a/archivist/marketplace/validation.nim b/archivist/marketplace/validation.nim index 4f4a95dc..94791227 100644 --- a/archivist/marketplace/validation.nim +++ b/archivist/marketplace/validation.nim @@ -42,8 +42,11 @@ proc waitUntilNextPeriod(validation: Validation) {.async.} = let period = validation.getCurrentPeriod() let periodicity = validation.marketplace.periodicity let periodEnd = periodicity.periodEnd(period) - trace "Waiting until next period", currentPeriod = period - await validation.clock.waitUntil((periodEnd + 1).toSecondsSince1970) + let targetTime = (periodEnd + 1).toSecondsSince1970 + trace "Waiting until next period", + currentPeriod = period, periodEnd, targetTime, now = validation.clock.now() + await validation.clock.waitUntil(targetTime) + trace "Finished waiting for next period", now = validation.clock.now() func groupIndexForSlotId*(slotId: SlotId, validationGroups: ValidationGroups): uint16 = let a = slotId.toArray diff --git a/archivist/marketplacestorage.nim b/archivist/marketplacestorage.nim index e9949ae9..9abcf4f0 100644 --- a/archivist/marketplacestorage.nim +++ b/archivist/marketplacestorage.nim @@ -37,3 +37,8 @@ method updateSlotExpiry*( ): Future[?!void] {.async: (raises: [CancelledError]).} = # TODO: update only the slot expiry, not the expiry of the entiry dataset await storage.node.updateExpiry(cid, expiry.toSecondsSince1970) + +method deleteSlot*( + storage: MarketplaceStorage, cid: Cid, slotIndex: uint64 +): Future[?!void] {.async: (raises: [CancelledError]).} = + await storage.node.deleteSlot(cid, slotIndex) diff --git a/archivist/merkletree/archivist/archivist.nim b/archivist/merkletree/archivist/archivist.nim index 9efd4cbf..872ce728 100644 --- a/archivist/merkletree/archivist/archivist.nim +++ b/archivist/merkletree/archivist/archivist.nim @@ -48,7 +48,7 @@ type mcodec*: MultiCodec # CodeHashes is not exported from libp2p -# So we need to recreate it instead of +# So we need to recreate it instead of proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} = for item in HashesList: result[item.mcodec] = item diff --git a/archivist/merkletree/archivist/coders.nim b/archivist/merkletree/archivist/coders.nim index bbc70deb..875769d4 100644 --- a/archivist/merkletree/archivist/coders.nim +++ b/archivist/merkletree/archivist/coders.nim @@ -33,7 +33,10 @@ proc encode*(self: ArchivistTree): seq[byte] = pb.finish pb.buffer -proc decode*(_: type ArchivistTree, data: seq[byte]): ?!ArchivistTree = +proc decode*(_: type ArchivistTree, data: openArray[byte]): ?!ArchivistTree = + if data.len == 0: + return success nil.ArchivistTree + var pb = initProtoBuffer(data) var mcodecCode: uint64 var leavesCount: uint64 @@ -57,6 +60,9 @@ proc decode*(_: type ArchivistTree, data: seq[byte]): ?!ArchivistTree = ArchivistTree.fromNodes(mcodec, nodes, leavesCount.int) proc encode*(self: ArchivistProof): seq[byte] = + if self.isNil: + return @[] + var pb = initProtoBuffer() pb.write(1, self.mcodec.uint64) pb.write(2, self.index.uint64) @@ -71,7 +77,10 @@ proc encode*(self: ArchivistProof): seq[byte] = pb.finish pb.buffer -proc decode*(_: type ArchivistProof, data: seq[byte]): ?!ArchivistProof = +proc decode*(_: type ArchivistProof, data: openArray[byte]): ?!ArchivistProof = + if data.len == 0: + return success(ArchivistProof(nil)) + var pb = initProtoBuffer(data) var mcodecCode: uint64 var index: uint64 diff --git a/archivist/merkletree/merkletree.nim b/archivist/merkletree/merkletree.nim index 134075a2..1e3c0af7 100644 --- a/archivist/merkletree/merkletree.nim +++ b/archivist/merkletree/merkletree.nim @@ -18,11 +18,13 @@ import ../errors type CompressFn*[H, K] = proc(x, y: H, key: K): ?!H {.noSideEffect, raises: [].} + # TODO: Make object, not ref MerkleTree*[H, K] = ref object of RootObj layers*: seq[seq[H]] compress*: CompressFn[H, K] zero*: H + # TODO: Make object, not ref MerkleProof*[H, K] = ref object of RootObj index*: int # linear index of the leaf, starting from 0 path*: seq[H] # order: from the bottom to the top diff --git a/archivist/metrics.nim b/archivist/metrics.nim new file mode 100644 index 00000000..7e1fb268 --- /dev/null +++ b/archivist/metrics.nim @@ -0,0 +1,36 @@ +## Copyright (c) 2025 Archivist Authors +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. + +{.push raises: [].} + +import pkg/chronicles +import pkg/metrics + +logScope: + topics = "archivist metrics" + +# Upload pipeline metrics +declarePublicHistogram( + archivist_upload_batch_flush_duration_seconds, + "Time to flush a single batch", + buckets = [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0], +) + +declarePublicCounter(archivist_upload_blocks_total, "Total number of blocks uploaded") + +declarePublicCounter(archivist_upload_batches_total, "Total number of batches flushed") + +declarePublicCounter(archivist_upload_bytes_total, "Total bytes uploaded") + +declarePublicHistogram( + archivist_upload_tree_build_duration_seconds, + "Time to build Merkle tree", + buckets = [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0], +) + +declarePublicGauge( + archivist_upload_active_batches, "Current number of in-flight batches" +) diff --git a/archivist/namespaces.nim b/archivist/namespaces.nim index af2ad6f1..37c3fada 100644 --- a/archivist/namespaces.nim +++ b/archivist/namespaces.nim @@ -17,11 +17,12 @@ const ArchivistBlocksNamespace* = ArchivistRepoNamespace & "/blocks" # blocks namespace ArchivistManifestNamespace* = ArchivistRepoNamespace & "/manifests" # manifest namespace - ArchivistBlocksTtlNamespace* = # Cid TTL - ArchivistMetaNamespace & "/ttl" - ArchivistBlockProofNamespace* = # Cid and Proof - ArchivistMetaNamespace & "/proof" + ArchivistBlocksMetaNamespace* = # Block metadata namespace + ArchivistMetaNamespace & "/blocks" + ArchivistBlockLeafNamespace* = # Cid and Proof + ArchivistMetaNamespace & "/leafs" ArchivistDhtNamespace* = "dht" # Dht namespace ArchivistDhtProvidersNamespace* = # Dht providers namespace ArchivistDhtNamespace & "/providers" ArchivistQuotaNamespace* = ArchivistMetaNamespace & "/quota" # quota's namespace + ArchivistOverlayNamespace* = ArchivistMetaNamespace & "/overlays" diff --git a/archivist/node.nim b/archivist/node.nim index d9497f85..fc0c502b 100644 --- a/archivist/node.nim +++ b/archivist/node.nim @@ -11,11 +11,11 @@ import std/options import std/sequtils -import std/strformat import std/sugar import times import pkg/taskpools +import pkg/stew/bitseqs import pkg/questionable import pkg/questionable/results import pkg/chronos @@ -30,6 +30,7 @@ import pkg/libp2p/stream/bufferstream import pkg/libp2p/routing_record import pkg/libp2p/signed_envelope +import ./metrics import ./chunker import ./slots import ./clock @@ -55,13 +56,17 @@ export logutils logScope: topics = "archivist node" -const DefaultFetchBatch = 10 +const + DefaultFetchBatch = 10 + DefaultStoreBatch* = 1024 ## Number of blocks to batch when storing data + MaxInFlightBatches = 4 ## Maximum concurrent batch flushes for bounded parallelism type ArchivistNode* = object switch: Switch networkId: PeerId networkStore: NetworkStore + repoStore: RepoStore engine: BlockExcEngine prover: ?Prover discovery: Discovery @@ -100,23 +105,6 @@ func `marketplace=`*(self: ArchivistNodeRef, marketplace: MarketplaceNode) = doAssert self.marketplace.isNone self.marketplace = some marketplace -proc storeManifest*( - self: ArchivistNodeRef, manifest: Manifest -): Future[?!bt.Block] {.async: (raises: [CancelledError]).} = - without encodedVerifiable =? manifest.encode(), err: - trace "Unable to encode manifest" - return failure(err) - - without blk =? bt.Block.new(data = encodedVerifiable, codec = ManifestCodec), error: - trace "Unable to create block from manifest" - return failure(error) - - if err =? (await self.networkStore.putBlock(blk)).errorOption: - trace "Unable to store manifest block", cid = blk.cid, err = err.msg - return failure(err) - - success blk - proc fetchManifest*( self: ArchivistNodeRef, cid: Cid ): Future[?!Manifest] {.async: (raises: [CancelledError]).} = @@ -128,7 +116,7 @@ proc fetchManifest*( trace "Retrieving manifest for cid", cid - without blk =? await self.networkStore.getBlock(BlockAddress.init(cid)), err: + without blk =? await self.networkStore.getBlock(cid), err: trace "Error retrieve manifest block", cid, err = err.msg return failure err @@ -142,19 +130,6 @@ proc fetchManifest*( return manifest.success -proc fetchManifest*( - self: ArchivistNodeRef, cid: Cid, expiry: SecondsSince1970 -): Future[?!Manifest] {.async: (raises: [CancelledError]).} = - without manifest =? await self.fetchManifest(cid), error: - trace "Unable to fetch manifest for cid", cid - return failure(error) - - if err =? (await self.networkStore.ensureExpiry(cid, expiry)).errorOption: - error "Failed to update manifest block expiry", cid, expiry - return failure(err) - - return success(manifest) - proc findPeer*(self: ArchivistNodeRef, peerId: PeerId): Future[?PeerRecord] {.async.} = ## Find peer using the discovery service from the given ArchivistNode ## @@ -165,28 +140,20 @@ proc connect*( ): Future[void] = self.switch.connect(peerId, addrs) +proc updateExpiry*( + self: ArchivistNodeRef, manifest: Manifest, expiry: SecondsSince1970 +): Future[?!void] {.async: (raises: [CancelledError]).} = + ?await self.repoStore.putOverlay(manifest.treeCid, expiry = expiry) + return success() + proc updateExpiry*( self: ArchivistNodeRef, manifestCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async: (raises: [CancelledError]).} = - without manifest =? await self.fetchManifest(manifestCid, expiry), error: + without manifest =? await self.fetchManifest(manifestCid), error: trace "Unable to fetch manifest for cid", manifestCid return failure(error) - try: - let ensuringFutures = Iter[int].new(0 ..< manifest.blocksCount).mapIt( - self.networkStore.ensureExpiry(manifest.treeCid, it, expiry) - ) - - let res = await allFinishedFailed[?!void](ensuringFutures) - if res.failure.len > 0: - trace "Some blocks failed to update expiry", len = res.failure.len - return failure("Some blocks failed to update expiry (" & $res.failure.len & " )") - except CancelledError as exc: - raise exc - except CatchableError as exc: - return failure(exc.msg) - - return success() + await self.updateExpiry(manifest, expiry) proc fetchBatched*( self: ArchivistNodeRef, @@ -199,41 +166,48 @@ proc fetchBatched*( ## Fetch blocks in batches of `batchSize` ## - # TODO: doesn't work if callee is annotated with async - # let - # iter = iter.map( - # (i: int) => self.networkStore.getBlock(BlockAddress.init(cid, i)) - # ) - - while not iter.finished: - let blockFutures = collect: - for i in 0 ..< batchSize: - if not iter.finished: - let address = BlockAddress.init(cid, iter.next()) - if not (await address in self.networkStore) or fetchLocal: - self.networkStore.getBlock(address) - - if blockFutures.len == 0: - continue - - without blockResults =? await allFinishedValues[?!bt.Block](blockFutures), err: - trace "Some blocks failed to fetch", err = err.msg - return failure(err) - - let blocks = blockResults.filterIt(it.isSuccess()).mapIt(it.value) - - let numOfFailedBlocks = blockResults.len - blocks.len - if numOfFailedBlocks > 0: - return - failure("Some blocks failed (Result) to fetch (" & $numOfFailedBlocks & ")") + await self.repoStore.withOverlay( + cid, + status = Downloading.some, + body = proc(): Future[?!void] {.closure, gcsafe, async: (raises: [CancelledError]).} = + # When fetchLocal = false, fetch bitmap once to filter already-present blocks + # in memory. This avoids O(N) per-block SQLite lookups (hasBlock -> + # getBlocksBitmap) and replaces them with a single bitmap fetch upfront. + # When fetchLocal = true we include all indices regardless, so no check needed. + let bits = + if not fetchLocal: + ?await self.repoStore.getBlocksBitmap(cid) + else: + BitSeq.init(0) + + while not iter.finished: + var batchIndices: seq[Natural] + for i in 0 ..< batchSize: + if not iter.finished: + let idx = iter.next() + # Include index if fetchLocal is set, or if it's not yet in the bitmap + if fetchLocal or idx >= bits.len or not bits[idx]: + batchIndices.add(idx.Natural) + + if batchIndices.len == 0: + continue + + without blocks =? (await self.networkStore.getBlocks(cid, batchIndices)), err: + trace "Some blocks failed to fetch", err = err.msg + return failure(err) - if not onBatch.isNil and batchErr =? (await onBatch(blocks)).errorOption: - return failure(batchErr) + if blocks.len != batchIndices.len: + return failure( + "Some blocks failed to fetch (" & $(batchIndices.len - blocks.len) & + " missing)" + ) - if not iter.finished: - await sleepAsync(1.millis) + if not onBatch.isNil and + batchErr =? (await onBatch(blocks.mapIt(it[1]))).errorOption: + return failure(batchErr) - success() + success(), + ) proc fetchBatched*( self: ArchivistNodeRef, @@ -271,7 +245,18 @@ proc fetchDatasetAsyncTask*(self: ArchivistNodeRef, manifest: Manifest) = ## Start fetching a dataset in the background. ## The task will be tracked and cleaned up on node shutdown. ## - self.trackedFutures.track(self.fetchDatasetAsync(manifest, fetchLocal = false)) + proc run(): Future[void] {.async: (raises: []).} = + try: + if err =? ( + await self.fetchBatched( + manifest = manifest, batchSize = DefaultFetchBatch, fetchLocal = false + ) + ).errorOption: + error "Unable to fetch dataset", err = err.msg + except CancelledError as exc: + trace "Dataset fetch cancelled", exc = exc.msg + + self.trackedFutures.track(run()) proc streamSingleBlock( self: ArchivistNodeRef, cid: Cid @@ -282,7 +267,7 @@ proc streamSingleBlock( let stream = BufferStream.new() - without blk =? (await self.networkStore.getBlock(BlockAddress.init(cid))), err: + without blk =? (await self.networkStore.getBlock(cid)), err: return failure(err) proc streamOneBlock(): Future[void] {.async: (raises: []).} = @@ -308,21 +293,36 @@ proc streamEntireDataset( var jobs: seq[Future[void]] let stream = LPStream(StoreStream.new(self.networkStore, manifest, pad = false)) if manifest.protected: - # Retrieve, decode and save to the local store all EС groups + # For protected manifests, erasure.decode owns the overlay lifecycle + # via its own withOverlay(treeCid, Storing) call. It also fetches all + # blocks via networkStore.getBlock, so no separate fetch job is needed. proc erasureJob(): Future[void] {.async: (raises: []).} = try: - # Spawn an erasure decoding job let erasure = Erasure.new( - self.networkStore, leoEncoderProvider, leoDecoderProvider, self.taskpool + self.networkStore, self.repoStore, leoEncoderProvider, leoDecoderProvider, + self.taskpool, ) - without _ =? (await erasure.decode(manifest)), error: - error "Unable to erasure decode manifest", manifestCid, exc = error.msg + + if err =? (await erasure.decode(manifest)).errorOption: + error "Unable to erasure decode manifest", manifestCid, exc = err.msg + return except CatchableError as exc: trace "Error erasure decoding manifest", manifestCid, exc = exc.msg jobs.add(erasureJob()) + else: + proc fetchJob(): Future[void] {.async: (raises: []).} = + try: + if err =? ( + await self.fetchBatched( + manifest = manifest, batchSize = DefaultFetchBatch, fetchLocal = false + ) + ).errorOption: + error "Unable to fetch blocks", err = err.msg + except CancelledError as exc: + trace "Cancelled fetching blocks", exc = exc.msg - jobs.add(self.fetchDatasetAsync(manifest, fetchLocal = false)) + jobs.add(fetchJob()) # Monitor stream completion and cancel background jobs when done proc monitorStream() {.async: (raises: []).} = @@ -355,6 +355,9 @@ proc retrieve*( return await self.streamSingleBlock(cid) + # track manifest CID in overlay for cleanup + ?await self.repoStore.putOverlay(manifest.treeCid, manifestCid = cid.some) + await self.streamEntireDataset(manifest, cid) proc deleteSingleBlock( @@ -373,32 +376,18 @@ proc deleteEntireDataset( # Deletion is a strictly local operation var store = self.networkStore.localStore - if not (await cid in store): - # As per the contract for delete*, an absent dataset is not an error. - return success() - without manifestBlock =? await store.getBlock(cid), err: return failure(err) without manifest =? Manifest.decode(manifestBlock), err: return failure(err) - let runtimeQuota = initDuration(milliseconds = 100) - var lastIdle = getTime() - for i in 0 ..< manifest.blocksCount: - if (getTime() - lastIdle) >= runtimeQuota: - await idleAsync() - lastIdle = getTime() - - if err =? (await store.delBlock(manifest.treeCid, i)).errorOption: - # The contract for delBlock is fuzzy, but we assume that if the block is - # simply missing we won't get an error. This is a best effort operation and - # can simply be retried. - error "Failed to delete block within dataset", index = i, err = err.msg - return failure(err) + if err =? (await self.repoStore.dropOverlay(manifest.treeCid)).errorOption: + error "Error dropping manifest overlay", cid, err = err.msg + return failure(err) if err =? (await store.delBlock(cid)).errorOption: - error "Error deleting manifest block", err = err.msg + warn "Manifest block already removed", cid, err = err.msg success() @@ -426,10 +415,13 @@ proc store*( filename: ?string = string.none, mimetype: ?string = string.none, blockSize = DefaultBlockSize, + storeBatchSize = DefaultStoreBatch, ): Future[?!Cid] {.async: (raises: [CancelledError]).} = ## Save stream contents as dataset with given blockSize ## to nodes's BlockStore, and return Cid of its manifest ## + ## Blocks are batched for efficient storage (storeBatchSize blocks per batch). + ## info "Storing data" let @@ -437,44 +429,129 @@ proc store*( dataCodec = BlockCodec chunker = LPStreamChunker.new(stream, chunkSize = blockSize) - var cids: seq[Cid] - - try: - while (let chunk = await chunker.getBytes(); chunk.len > 0): - without mhash =? MultiHash.digest($hcodec, chunk).mapFailure, err: - return failure(err) - - without cid =? Cid.init(CIDv1, dataCodec, mhash).mapFailure, err: - return failure(err) - - without blk =? bt.Block.new(cid, chunk, verify = false): - return failure("Unable to init block from chunk!") - - cids.add(cid) - - if err =? (await self.networkStore.putBlock(blk)).errorOption: - error "Unable to store block", cid = blk.cid, err = err.msg - return failure(&"Unable to store block {blk.cid}") - except CancelledError as exc: - raise exc - except CatchableError as exc: - return failure(exc.msg) - finally: + defer: await stream.close() - without tree =? ArchivistTree.init(cids), err: - return failure(err) - - without treeCid =? tree.rootCid(CIDv1, dataCodec), err: - return failure(err) + proc flushBatch( + tmpCid: Cid, batch: sink seq[(bt.Block, Natural)] + ): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Flush a batch of blocks to storage using batched putBlocks + if batch.len == 0: + return success() + + let batchStart = Moment.now() + trace "Flushing block batch", count = batch.len + + # Convert to the format expected by putBlocks: (Block, Natural, ArchivistProof) + # We don't have proofs yet (built after tree construction), so use nil + var items: seq[(bt.Block, Natural, ArchivistProof)] + for (blk, idx) in batch: + items.add((blk, idx, nil)) + + ?await self.repoStore.putBlocks(tmpCid, items) + let batchDone = Moment.now() + trace "Batch flush complete", + duration = $(batchDone - batchStart), items = items.len + + archivist_upload_batch_flush_duration_seconds.observe( + (batchDone - batchStart).milliseconds.float64 / 1000.0 + ) + success() + + let treeCid = + ?await self.repoStore.withTmpOverlay( + body = proc( + tmpCid: Cid + ): Future[?!Cid] {.closure, gcsafe, async: (raises: [CancelledError]).} = + var + index = 0 + cids: seq[Cid] + inFlight: seq[Future[?!void]] ## Track in-flight batch flushes + blockBatch: seq[(bt.Block, Natural)] ## (block, index) pairs for batching + + proc fireBoundedBatch( + batch: seq[(bt.Block, Natural)] + ): Future[?!void] {.async: (raises: [CancelledError]).} = + # wait if at capacity before launching new batch + if inFlight.len >= MaxInFlightBatches: + # Remove first future and await it (cleanup before await to avoid leak) + let fut = ?catchAsync(await one(inFlight)) + inFlight.keepItIf(FutureBase(it) != FutureBase(fut)) + ?catchAsync(?await fut) + + # Launch batch flush without awaiting (adds to window) + var batch = batch + inFlight.add(flushBatch(tmpCid, move batch)) + archivist_upload_batches_total.inc() + archivist_upload_active_batches.set(inFlight.len.int64) + + success() + + defer: + if inFlight.len > 0: + warn "Early exit, cancelling outstanding upload batches", + batches = inFlight.len + await allFutures(inFlight.mapIt(it.cancelAndWait())) + + while true: + let chunk = ?await chunker.getBytes() + archivist_upload_bytes_total.inc(chunk.len.int64) + if chunk.len == 0: + trace "Chunker finished reading stream", read = NBytes(chunker.offset) + break + + let + mhash = ?MultiHash.digest($hcodec, chunk).mapFailure + cid = ?Cid.init(CIDv1, dataCodec, mhash).mapFailure + blk = ?bt.Block.new(cid, chunk, verify = false) + + archivist_upload_blocks_total.inc() + cids.add(cid) + blockBatch.add((blk, index.Natural)) + index.inc + + # Flush batch when full + if blockBatch.len >= storeBatchSize: + archivist_upload_active_batches.set(inFlight.len.int64) + ?await fireBoundedBatch(blockBatch) + archivist_upload_active_batches.set(inFlight.len.int64) + blockBatch.setLen(0) + + # Flush batch on last iteration + if blockBatch.len > 0: + ?await fireBoundedBatch(blockBatch) + blockBatch.setLen(0) + + await allFutures(inFlight) + # return the first failed fut + discard inFlight.mapIt(?catch(it.read)) + inFlight.setLen(0) + + let + treeStart = Moment.now() + tree = ?ArchivistTree.init(cids) + treeCid = ?tree.rootCid(CIDv1, dataCodec) + treeDone = Moment.now() + archivist_upload_tree_build_duration_seconds.observe( + (treeDone - treeStart).milliseconds.float64 / 1000.0 + ) - for index, cid in cids: - without proof =? tree.getProof(index), err: - return failure(err) - if err =? - (await self.networkStore.putCidAndProof(treeCid, index, cid, proof)).errorOption: - # TODO add log here - return failure(err) + # TODO: Once we have progressive tree building we can get rid of the + # separate proofs putting and just put leafs directly in the same + # putBlock call as we build the tree + var proofItems: seq[(Natural, Cid, ArchivistProof)] + for index, cid in cids: + proofItems.add((index.Natural, cid, ?tree.getProof(index))) + if proofItems.len >= storeBatchSize: + ?await self.repoStore.putCidsAndProofs(tmpCid, proofItems) + proofItems.setLen(0) + + if proofItems.len > 0: + ?await self.repoStore.putCidsAndProofs(tmpCid, proofItems) + proofItems.setLen(0) + + success treeCid + ) let manifest = Manifest.new( treeCid = treeCid, @@ -487,9 +564,11 @@ proc store*( mimetype = mimetype, ) - without manifestBlk =? await self.storeManifest(manifest), err: - error "Unable to store manifest" - return failure(err) + # store the manifest + let manifestBlk = ?await self.repoStore.storeManifest(manifest) + + # track manifest CID in overlay for cleanup + ?await self.repoStore.putOverlay(treeCid, manifestCid = manifestBlk.cid.some) info "Stored data", manifestCid = manifestBlk.cid, @@ -536,11 +615,15 @@ proc ensureProtectedManifest( return success manifest # Erasure code the dataset according to provided parameters - let erasure = Erasure.new( - self.networkStore.localStore, leoEncoderProvider, leoDecoderProvider, self.taskpool - ) + let + erasure = Erasure.new( + self.networkStore.localStore, self.repoStore, leoEncoderProvider, + leoDecoderProvider, self.taskpool, + ) + encodedManifest = ?await erasure.encode(manifest, ecK, ecM) + manifestBlk = ?await self.repoStore.storeManifest(encodedManifest) - return await erasure.encode(manifest, ecK, ecM) + success encodedManifest proc ensureVerifiableManifest( self: ArchivistNodeRef, manifest: Manifest, ecK: uint, ecM: uint @@ -552,7 +635,8 @@ proc ensureVerifiableManifest( return success protected # Create verifiable manifest from protected manifest - let builder = ?Poseidon2Builder.new(self.networkStore.localStore, protected) + let builder = + ?Poseidon2Builder.new(self.networkStore.localStore, self.repoStore, protected) return await builder.buildManifest() proc setupRequest( @@ -590,7 +674,7 @@ proc setupRequest( let manifest = ?await self.fetchManifest(cid) verifiable = ?await self.ensureVerifiableManifest(manifest, ecK, ecM) - manifestBlk = ?await self.storeManifest(verifiable) + manifestBlk = ?await self.repoStore.storeManifest(verifiable) verifyRoot = (?verifiable.verifyRoot.fromVerifyCid).toBytes slotBytes = (verifiable.blockSize.int * verifiable.numSlotBlocks).NBytes @@ -678,8 +762,7 @@ proc storeSlot*( slotSize = slotSize trace "Received a request to store a slot" - - without manifest =? (await self.fetchManifest(cid, expiry)), err: + without manifest =? (await self.fetchManifest(cid)), err: error "Unable to fetch manifest for cid", cid, err = err.msg return failure(err) @@ -687,8 +770,17 @@ proc storeSlot*( error "Validation of verifiable manifest failed", err = err.msg return failure(err) + # track manifest CID in overlay for cleanup + ?await self.repoStore.putOverlay(manifest.treeCid, manifestCid = cid.some) + + if err =? (await self.updateExpiry(manifest, expiry)).errorOption: + error "Unable to update manifest expiry", cid, err = err.msg + return failure(err) + without builder =? - Poseidon2Builder.new(self.networkStore, manifest, manifest.verifiableStrategy), err: + Poseidon2Builder.new( + self.networkStore, self.repoStore, manifest, manifest.verifiableStrategy + ), err: error "Unable to create slots builder", err = err.msg return failure(err) @@ -696,21 +788,6 @@ proc storeSlot*( error "Slot index not in manifest" return failure(newException(ArchivistError, "Slot index not in manifest")) - proc updateExpiry( - blocks: seq[bt.Block] - ): Future[?!void] {.async: (raises: [CancelledError]).} = - trace "Updating expiry for blocks", blocks = blocks.len - - let ensureExpiryFutures = - blocks.mapIt(self.networkStore.ensureExpiry(it.cid, expiry)) - - let res = await allFinishedFailed[?!void](ensureExpiryFutures) - if res.failure.len > 0: - error "Some blocks failed to update expiry", len = res.failure.len - return failure("Some blocks failed to update expiry (" & $res.failure.len & " )") - - return success() - if slotIndex > int.high.uint64: error "Cannot cast slot index to int", slotIndex = slotIndex return failure(newException(ArchivistError, "Cannot cast slot index to int")) @@ -721,32 +798,22 @@ proc storeSlot*( if repair: trace "start repairing slot", slotIdx - try: - let erasure = Erasure.new( - self.networkStore, leoEncoderProvider, leoDecoderProvider, self.taskpool - ) - if err =? (await erasure.repair(manifest)).errorOption: - error "Unable to erasure decode repairing manifest", - cid = manifest.treeCid, exc = err.msg - return failure(err) + let erasure = Erasure.new( + self.networkStore, self.repoStore, leoEncoderProvider, leoDecoderProvider, + self.taskpool, + ) + if err =? (await erasure.repair(manifest)).errorOption: + error "Unable to erasure decode repairing manifest", + cid = manifest.treeCid, exc = err.msg + return failure(err) - # Iterate the slot blocks. Provide them to the updateExpiry callback. - while not blksIter.finished: - without blk =? - await self.networkStore.getBlock(manifest.treeCid, blksIter.next()), err: - error "Unable to get slot block after repair" - return failure(err) - if err =? (await updateExpiry(@[blk])).errorOption: - error "Unable to update expiry for slot block after repair" - return failure(err) - except CatchableError as exc: - error "Error erasure decoding repairing manifest", - cid = manifest.treeCid, exc = exc.msg - return failure(exc.msg) + while not blksIter.finished: + without blk =? await self.networkStore.getBlock(manifest.treeCid, blksIter.next()), + err: + error "Unable to get slot block after repair" + return failure(err) else: - if err =? ( - await self.fetchBatched(manifest.treeCid, blksIter, onBatch = updateExpiry) - ).errorOption: + if err =? (await self.fetchBatched(manifest.treeCid, blksIter)).errorOption: error "Unable to fetch blocks", err = err.msg return failure(err) @@ -782,7 +849,9 @@ proc proveSlot*( let manifest = ?await self.fetchManifest(cid) builder = - ?Poseidon2Builder.new(self.networkStore, manifest, manifest.verifiableStrategy) + ?Poseidon2Builder.new( + self.networkStore, self.repoStore, manifest, manifest.verifiableStrategy + ) sampler = ?Poseidon2Sampler.new(slotIdx, self.networkStore, builder) when defined(verify_circuit): @@ -804,6 +873,43 @@ proc proveSlot*( warn "Prover not enabled" failure "Prover not enabled" +proc deleteSlot*( + self: ArchivistNodeRef, cid: Cid, slotIdx: uint64 +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Handle slot failure - mark overlay as failed so maintenance will drop it + ## + logScope: + cid = $cid + slot = slotIdx + + trace "Marking slot as failed" + + let manifest = ?await self.fetchManifest(cid) + if not manifest.verifiable: + warn "Attempting to fail a slot with a non-verifiable manifest", cid, slotIdx + + let slotCid = manifest.slotRoots[slotIdx] + + # Mark slot overlay as Failure so maintenance will drop it and cleanup manifest + if err =? ( + await self.repoStore.putOverlay( + slotCid, status = OverlayStatus.Failure.some, manifestCid = cid.some + ) + ).errorOption: + warn "Error marking slot overlay failed", err = err.msg + + # Mark tree overlay as Failure so maintenance will drop it and cleanup manifest + if err =? ( + await self.repoStore.putOverlay( + manifest.treeCid, status = OverlayStatus.Failure.some, manifestCid = cid.some + ) + ).errorOption: + warn "Error marking tree overlay failed", err = err.msg + + trace "Slot marked as failed" + + return success() + proc start*(self: ArchivistNodeRef) {.async.} = if not self.engine.isNil: await self.engine.start() @@ -839,6 +945,7 @@ proc new*( T: type ArchivistNodeRef, switch: Switch, networkStore: NetworkStore, + repoStore: RepoStore, engine: BlockExcEngine, discovery: Discovery, taskpool: Taskpool, @@ -851,6 +958,7 @@ proc new*( ArchivistNodeRef( switch: switch, networkStore: networkStore, + repoStore: repoStore, engine: engine, prover: prover, discovery: discovery, diff --git a/archivist/rest/api.nim b/archivist/rest/api.nim index 936f7c14..d3364453 100644 --- a/archivist/rest/api.nim +++ b/archivist/rest/api.nim @@ -45,6 +45,8 @@ logScope: declareCounter(archivist_api_uploads, "archivist API uploads") declareCounter(archivist_api_downloads, "archivist API downloads") +const DefaultStreamBatch* = 128 # Number of blocks to fetch per stream read + proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} = 0 @@ -123,7 +125,7 @@ proc retrieveCid( while not stream.atEof: var - buff = newSeqUninitialized[byte](DefaultBlockSize.int) + buff = newSeqUninitialized[byte](DefaultStreamBatch * DefaultBlockSize.int) len = await stream.readOnce(addr buff[0], buff.len) buff.setLen(len) @@ -286,8 +288,8 @@ proc initDataApi(node: ArchivistNodeRef, repoStore: RepoStore, router: var RestR cid: Cid, resp: HttpResponseRef ) -> RestApiResponse: ## Deletes either a single block or an entire dataset - ## from the local node. Does nothing and returns 204 - ## if the dataset is not locally available. + ## from the local node. Returns 404 if the dataset + ## is not locally available. ## var headers = buildCorsHeaders("DELETE", allowedOrigin) @@ -295,6 +297,8 @@ proc initDataApi(node: ArchivistNodeRef, repoStore: RepoStore, router: var RestR return RestApiResponse.error(Http400, $cid.error(), headers = headers) if err =? (await node.delete(cid.get())).errorOption: + if err of BlockNotFoundError: + return RestApiResponse.error(Http404, err.msg, headers = headers) return RestApiResponse.error(Http500, err.msg, headers = headers) if corsOrigin =? allowedOrigin: diff --git a/archivist/rng.nim b/archivist/rng.nim index f6e4def9..d668d9c8 100644 --- a/archivist/rng.nim +++ b/archivist/rng.nim @@ -12,6 +12,8 @@ import pkg/libp2p/crypto/crypto import pkg/bearssl/rand +export rand + type RngSampleError = object of CatchableError Rng* = ref HmacDrbgContext diff --git a/archivist/slots/builder/builder.nim b/archivist/slots/builder/builder.nim index 8c9d6ff6..e97048ca 100644 --- a/archivist/slots/builder/builder.nim +++ b/archivist/slots/builder/builder.nim @@ -21,11 +21,13 @@ import pkg/constantine/math/io/io_fields import ../../logutils import ../../utils +import ../../utils/poseidon2digest import ../../stores import ../../manifest import ../../merkletree import ../../utils/asynciter import ../../indexingstrategy +import ../../archivisttypes import ../converters @@ -35,7 +37,8 @@ logScope: topics = "archivist slotsbuilder" type SlotsBuilder*[SomeTree, SomeHash] = ref object of RootObj - store: BlockStore + networkStore: BlockStore + repoStore: RepoStore manifest: Manifest # current manifest strategy: IndexingStrategy # indexing strategy cellSize: NBytes # cell size @@ -131,12 +134,11 @@ func slotIndicesIter*[SomeTree, SomeHash]( func slotIndices*[SomeTree, SomeHash]( self: SlotsBuilder[SomeTree, SomeHash], slot: Natural -): seq[int] = +): ?!seq[int] = ## Returns the slot indices. ## - if iter =? self.strategy.getIndices(slot).catch: - return toSeq(iter) + success toSeq(?catch(self.strategy.getIndices(slot))) func manifest*[SomeTree, SomeHash](self: SlotsBuilder[SomeTree, SomeHash]): Manifest = ## Returns the manifest. @@ -158,24 +160,17 @@ proc buildBlockTree*[SomeTree, SomeHash]( cellSize = self.cellSize trace "Building block tree" - if slotPos > (self.manifest.numSlotBlocks - 1): # pad blocks are 0 byte blocks trace "Returning empty digest tree for pad block" return success (self.emptyBlock, self.emptyDigestTree) - without blk =? await self.store.getBlock(self.manifest.treeCid, blkIdx), e: - error "Failed to get block CID for tree at index", e = e.msg - return failure(e) - + let blk = ?await self.networkStore.getBlock(self.manifest.treeCid, blkIdx) if blk.isEmpty: - success (self.emptyBlock, self.emptyDigestTree) - else: - without tree =? SomeTree.digestTree(blk.data, self.cellSize.int), e: - error "Failed to create digest for block", e = e.msg - return failure(e) + return success (self.emptyBlock, self.emptyDigestTree) - success (blk.data, tree) + let tree = ?SomeTree.digestTree(blk.data, self.cellSize.int) + success (blk.data, tree) proc getCellHashes*[SomeTree, SomeHash]( self: SlotsBuilder[SomeTree, SomeHash], slotIndex: Natural @@ -196,20 +191,15 @@ proc getCellHashes*[SomeTree, SomeHash]( slotIndex = slotIndex let hashes = collect(newSeq): - for i, blkIdx in ?self.strategy.getIndices(slotIndex).catch: + for i, blkIdx in ?catch(self.strategy.getIndices(slotIndex)): logScope: blkIdx = blkIdx pos = i trace "Getting block CID for tree at index" - - without (_, tree) =? (await self.buildBlockTree(blkIdx, i)), e: - error "Failed to get block CID for tree at index", e = e.msg - return failure(e) - - without digest =? tree.root, e: - error "Failed to get block CID for tree at index", e = e.msg - return failure(e) + let + (_, tree) = ?await self.buildBlockTree(blkIdx, i) + digest = ?tree.root trace "Get block digest", digest = digest.toHex digest @@ -222,15 +212,7 @@ proc buildSlotTree*[SomeTree, SomeHash]( ## Build the slot tree from the block digest hashes ## and return the tree. - try: - without cellHashes =? (await self.getCellHashes(slotIndex)), e: - error "Failed to select slot blocks", e = e.msg - return failure(e) - - SomeTree.init(cellHashes) - except IndexingError as e: - error "Failed to build slot tree", e = e.msg - return failure(e) + SomeTree.init(?await self.getCellHashes(slotIndex)) proc buildSlot*[SomeTree, SomeHash]( self: SlotsBuilder[SomeTree, SomeHash], slotIndex: Natural @@ -251,19 +233,35 @@ proc buildSlot*[SomeTree, SomeHash]( return failure(e) trace "Storing slot tree", treeCid, slotIndex, leaves = tree.leavesCount - for i, leaf in tree.leaves: - without cellCid =? leaf.toCellCid, e: - error "Failed to get CID for slot cell", e = e.msg - return failure(e) + ?await self.repoStore.withOverlay( + treeCid, + status = Storing.some, + body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = + var proofItems: seq[(Natural, Cid, Cid, ArchivistProof)] + for i, blkIdx in ?self.slotIndicesIter(slotIndex): + without cellCid =? tree.leaves[i].toCellCid, e: + error "Failed to get CID for slot cell", e = e.msg + return failure(e) - without proof =? tree.getProof(i) and encodableProof =? proof.toEncodableProof, e: - error "Failed to get proof for slot tree", e = e.msg - return failure(e) + without proof =? tree.getProof(i) and encodableProof =? proof.toEncodableProof, + e: + error "Failed to get proof for slot tree", e = e.msg + return failure(e) + + # For pad blocks, use empty CID for blkCid since there's no real block + # TODO: make sure we do this as a batch, single get/put are very costly + # with CAS semantics + let blkCid = + if blkIdx >= self.manifest.numSlotBlocks: + ?emptyCid(self.manifest.version, self.manifest.hcodec, BlockCodec) + else: + ?await self.repoStore.getCid(self.manifest.treeCid, blkIdx) - if e =? - (await self.store.putCidAndProof(treeCid, i, cellCid, encodableProof)).errorOption: - error "Failed to store slot tree", e = e.msg - return failure(e) + proofItems.add((i.Natural, cellCid, blkCid, encodableProof)) + + ?await self.repoStore.putCellCidsAndProofs(treeCid, proofItems) + success(), + ) tree.root() @@ -287,10 +285,7 @@ proc buildSlots*[SomeTree, SomeHash]( if self.slotRoots.len == 0: self.slotRoots = collect(newSeq): for i in 0 ..< self.manifest.numSlots: - without slotRoot =? (await self.buildSlot(i)), e: - error "Failed to build slot", e = e.msg, index = i - return failure(e) - slotRoot + ?await self.buildSlot(i) without tree =? self.buildVerifyTree(self.slotRoots) and root =? tree.root, e: error "Failed to build slot roots tree", e = e.msg @@ -307,14 +302,11 @@ proc buildSlots*[SomeTree, SomeHash]( proc buildManifest*[SomeTree, SomeHash]( self: SlotsBuilder[SomeTree, SomeHash] ): Future[?!Manifest] {.async: (raises: [CancelledError]).} = - if e =? (await self.buildSlots()).errorOption: - error "Failed to build slot roots", e = e.msg - return failure(e) - - without rootCids =? self.slotRoots.toSlotCids(), e: - error "Failed to map slot roots to CIDs", e = e.msg - return failure(e) + ## Build the manifest with the slot roots and return it. + ## + ?await self.buildSlots() # build the slots + let rootCids = ?self.slotRoots.toSlotCids() without rootProvingCidRes =? self.verifyRoot .? toVerifyCid() and rootProvingCid =? rootProvingCidRes, e: error "Failed to map slot roots to CIDs", e = e.msg @@ -326,7 +318,8 @@ proc buildManifest*[SomeTree, SomeHash]( proc new*[SomeTree, SomeHash]( _: type SlotsBuilder[SomeTree, SomeHash], - store: BlockStore, + networkStore: BlockStore, + repoStore: RepoStore, manifest: Manifest, strategy = LinearStrategy, cellSize = DefaultCellSize, @@ -394,7 +387,8 @@ proc new*[SomeTree, SomeHash]( trace "Creating slots builder" var self = SlotsBuilder[SomeTree, SomeHash]( - store: store, + networkStore: networkStore, + repoStore: repoStore, manifest: manifest, strategy: strategy, cellSize: cellSize, diff --git a/archivist/slots/proofs/backends/circomcompat.nim b/archivist/slots/proofs/backends/circomcompat.nim index b01a6412..e3b04399 100644 --- a/archivist/slots/proofs/backends/circomcompat.nim +++ b/archivist/slots/proofs/backends/circomcompat.nim @@ -56,11 +56,13 @@ func normalizeInput*[SomeHash]( ## inputs to the CIRCOM circuit for testing and debugging when one wishes ## to bypass the Rust FFI. - let normSamples = collect: - for sample in input.samples: - var merklePaths = sample.merklePaths - merklePaths.setLen(self.slotDepth) + var normSamples: seq[Sample[SomeHash]] + for sample in input.samples: + var merklePaths = sample.merklePaths + merklePaths.setLen(self.slotDepth) + normSamples.add( Sample[SomeHash](cellData: sample.cellData, merklePaths: merklePaths) + ) var normSlotProof = input.slotProof normSlotProof.setLen(self.datasetDepth) diff --git a/archivist/slots/proofs/backends/converters.nim b/archivist/slots/proofs/backends/converters.nim index 76d3a8bc..11117914 100644 --- a/archivist/slots/proofs/backends/converters.nim +++ b/archivist/slots/proofs/backends/converters.nim @@ -65,8 +65,8 @@ func toG1*(g: NimGroth16G1): G1Point = x: array[32, byte] y: array[32, byte] - assert x.marshal(g.x, Endianness.littleEndian) - assert y.marshal(g.y, Endianness.littleEndian) + doAssert x.marshal(g.x, Endianness.littleEndian) + doAssert y.marshal(g.y, Endianness.littleEndian) G1Point(x: UInt256.fromBytesLE(x), y: UInt256.fromBytesLE(y)) @@ -75,10 +75,10 @@ func toG2*(g: NimGroth16G2): G2Point = x: array[2, array[32, byte]] y: array[2, array[32, byte]] - assert x[0].marshal(g.x.coords[0], Endianness.littleEndian) - assert x[1].marshal(g.x.coords[1], Endianness.littleEndian) - assert y[0].marshal(g.y.coords[0], Endianness.littleEndian) - assert y[1].marshal(g.y.coords[1], Endianness.littleEndian) + doAssert x[0].marshal(g.x.coords[0], Endianness.littleEndian) + doAssert x[1].marshal(g.x.coords[1], Endianness.littleEndian) + doAssert y[0].marshal(g.y.coords[0], Endianness.littleEndian) + doAssert y[1].marshal(g.y.coords[1], Endianness.littleEndian) G2Point( x: Fp2Element(real: UInt256.fromBytesLE(x[0]), imag: UInt256.fromBytesLE(x[1])), diff --git a/archivist/slots/sampler/sampler.nim b/archivist/slots/sampler/sampler.nim index 9706a332..7f9c7fc2 100644 --- a/archivist/slots/sampler/sampler.nim +++ b/archivist/slots/sampler/sampler.nim @@ -56,7 +56,7 @@ proc getSample*[SomeTree, SomeHash]( cellsPerBlock = self.builder.numBlockCells blkCellIdx = cellIdx.toCellInBlk(cellsPerBlock) # block cell index blkSlotIdx = cellIdx.toBlkInSlot(cellsPerBlock) # slot tree index - origBlockIdx = self.builder.slotIndices(self.index)[blkSlotIdx] + origBlockIdx = (?self.builder.slotIndices(self.index))[blkSlotIdx] # convert to original dataset block index logScope: diff --git a/archivist/stores.nim b/archivist/stores.nim index 91d2c786..554bc136 100644 --- a/archivist/stores.nim +++ b/archivist/stores.nim @@ -1,4 +1,3 @@ -import ./stores/cachestore import ./stores/blockstore import ./stores/networkstore import ./stores/repostore @@ -6,5 +5,4 @@ import ./stores/maintenance import ./stores/keyutils import ./stores/treehelper -export - cachestore, blockstore, networkstore, repostore, keyutils, treehelper, maintenance +export blockstore, networkstore, repostore, keyutils, treehelper, maintenance diff --git a/archivist/stores/blockstore.nim b/archivist/stores/blockstore.nim index 5457aec7..06338332 100644 --- a/archivist/stores/blockstore.nim +++ b/archivist/stores/blockstore.nim @@ -14,10 +14,10 @@ import pkg/libp2p import pkg/questionable import pkg/questionable/results -import ../clock import ../blocktype import ../merkletree import ../utils +import ../manifest export blocktype @@ -33,21 +33,96 @@ type BlockStore* = ref object of RootObj onBlockStored*: ?CidCallback +########################################################### +# Get methods +########################################################### + +method getBlocks*( + self: BlockStore, cids: seq[Cid] +): Future[?!seq[Block]] {.base, async: (raises: [CancelledError]), gcsafe.} = + ## Get multiple blocks by CID as a batch (no tree context). + ## Missing blocks are silently omitted from the result. + ## + + raiseAssert("getBlocks by cids not implemented!") + method getBlock*( self: BlockStore, cid: Cid ): Future[?!Block] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Get a block from the blockstore + ## Get a block from the blockstore by CID (no tree context). + ## Default: delegates to getBlocks batch. + ## + + let blocks = ?await self.getBlocks(@[cid]) + if blocks.len == 0: + return failure(newException(BlockNotFoundError, "Block not found")) + success(blocks[0]) + +method getBlocks*( + self: BlockStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Natural, Block)]] {.base, async: (raises: [CancelledError]), gcsafe.} = + ## Get multiple blocks by tree CID and indices as a batch ## - raiseAssert("getBlock by cid not implemented!") + raiseAssert("getBlocks by treeCid not implemented!") method getBlock*( self: BlockStore, treeCid: Cid, index: Natural ): Future[?!Block] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Get a block from the blockstore + ## Get a block by tree CID and index. + ## Default: delegates to getBlocks batch. + ## + + let blocks = ?await self.getBlocks(treeCid, @[index]) + if blocks.len == 0: + return failure(newException(BlockNotFoundError, "Block not found")) + success(blocks[0][1]) + +method getBlocksAndProofs*( + self: BlockStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Natural, Block, ArchivistProof)]] {. + base, async: (raises: [CancelledError]), gcsafe +.} = + ## Get multiple blocks and proofs by tree CID and indices as a batch + ## + + raiseAssert("getBlocksAndProofs by treeCid not implemented!") + +method getBlockAndProof*( + self: BlockStore, treeCid: Cid, index: Natural +): Future[?!(Natural, Block, ArchivistProof)] {. + base, async: (raises: [CancelledError]), gcsafe +.} = + ## Get a block and proof by tree CID and index. + ## Default: delegates to getBlocksAndProofs batch. + ## + + let results = ?await self.getBlocksAndProofs(treeCid, @[index]) + if results.len == 0: + return failure(newException(BlockNotFoundError, "Block not found")) + success(results[0]) + +method getCidsAndProofs*( + self: BlockStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Cid, ArchivistProof)]] {. + base, async: (raises: [CancelledError]), gcsafe +.} = + ## Get multiple CIDs and proofs as a batch + ## + + raiseAssert("getCidsAndProofs not implemented!") + +method getCidAndProof*( + self: BlockStore, treeCid: Cid, index: Natural +): Future[?!(Cid, ArchivistProof)] {.base, async: (raises: [CancelledError]), gcsafe.} = + ## Get a CID and proof by tree CID and index. + ## Default: delegates to getCidsAndProofs batch. ## - raiseAssert("getBlock by treecid not implemented!") + let results = ?await self.getCidsAndProofs(treeCid, @[index]) + if results.len == 0: + return failure(newException(BlockNotFoundError, "CID and proof not found")) + success(results[0]) method getCid*( self: BlockStore, treeCid: Cid, index: Natural @@ -57,84 +132,133 @@ method getCid*( raiseAssert("getCid by treecid not implemented!") -method getBlock*( - self: BlockStore, address: BlockAddress -): Future[?!Block] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Get a block from the blockstore - ## - - raiseAssert("getBlock by addr not implemented!") - method completeBlock*( - self: BlockStore, address: BlockAddress, blk: Block + self: BlockStore, treeCid: Cid, index: Natural, blk: Block ) {.base, gcsafe.} = discard -method getBlockAndProof*( - self: BlockStore, treeCid: Cid, index: Natural -): Future[?!(Block, ArchivistProof)] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Get a block and associated inclusion proof by Cid of a merkle tree and an index of a leaf in a tree - ## - - raiseAssert("getBlockAndProof not implemented!") +########################################################### +# Put methods +########################################################### method putBlock*( self: BlockStore, blk: Block, ttl = Duration.none -): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Put a block to the blockstore +): Future[?!void] {. + base, + gcsafe, + async: (raises: [CancelledError]), + deprecated: "Use putBlocks(treeCid, seq[(Natural, Block, ArchivistProof)])" +.} = + ## Put a block to the blockstore (deprecated) ## raiseAssert("putBlock not implemented!") -method putCidAndProof*( - self: BlockStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: ArchivistProof +method putBlocks*( + self: BlockStore, treeCid: Cid, items: seq[(Block, Natural, ArchivistProof)] +): Future[?!void] {.base, async: (raises: [CancelledError]).} = + ## Put multiple leafs and blocks as a batch (primary method) + ## + + raiseAssert("putBlocks not implemented!") + +method putBlock*( + self: BlockStore, treeCid: Cid, blk: Block, index: Natural, proof: ArchivistProof +): Future[?!void] {.base, async: (raises: [CancelledError]).} = + ## Put a single leaf and block. + ## Default: delegates to putBlocks batch. + ## + + await self.putBlocks(treeCid, @[(blk, index, proof)]) + +method putBlocks*( + self: BlockStore, treeCid: Cid, blocks: seq[(Natural, Block)] ): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Put a block proof to the blockstore + ## Put multiple blocks without proofs as a batch ## - raiseAssert("putCidAndProof not implemented!") + raiseAssert("putBlocks not implemented!") -method getCidAndProof*( - self: BlockStore, treeCid: Cid, index: Natural -): Future[?!(Cid, ArchivistProof)] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Get a block proof from the blockstore +method putBlock*( + self: BlockStore, treeCid: Cid, blk: Block, index: Natural +): Future[?!void] {.base, async: (raises: [CancelledError]).} = + ## Put a single block without proof. + ## Default: delegates to putBlocks(treeCid, seq[(Natural, Block)]) batch. ## - raiseAssert("getCidAndProof not implemented!") + await self.putBlocks(treeCid, @[(index, blk)]) -method ensureExpiry*( - self: BlockStore, cid: Cid, expiry: SecondsSince1970 +method putCidsAndProofs*( + self: BlockStore, treeCid: Cid, items: seq[(Natural, Cid, ArchivistProof)] ): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Ensure that block's assosicated expiry is at least given timestamp - ## If the current expiry is lower then it is updated to the given one, otherwise it is left intact + ## Put multiple CIDs and proofs as a batch ## - raiseAssert("Not implemented!") + raiseAssert("putCidsAndProofs not implemented!") -method ensureExpiry*( - self: BlockStore, treeCid: Cid, index: Natural, expiry: SecondsSince1970 +method putCidAndProof*( + self: BlockStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: ArchivistProof ): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Ensure that block's associated expiry is at least given timestamp - ## If the current expiry is lower then it is updated to the given one, otherwise it is left intact + ## Put a CID and proof. + ## Default: delegates to putCidsAndProofs batch. ## - raiseAssert("Not implemented!") + await self.putCidsAndProofs(treeCid, @[(index, blockCid, proof)]) + +method putCellCidsAndProofs*( + self: BlockStore, treeCid: Cid, items: seq[(Natural, Cid, Cid, ArchivistProof)] +): Future[?!void] {.base, async: (raises: [CancelledError]).} = + ## Put multiple cell CIDs and proofs as a batch + ## + + raiseAssert("putCellCidsAndProofs not implemented!") + +method putCellCidAndProof*( + self: BlockStore, + treeCid: Cid, + cellCid: Cid, + blkCid: Cid, + index: Natural, + proof: ArchivistProof, +): Future[?!void] {.base, async: (raises: [CancelledError]).} = + ## Put a single cell CID and proof. + ## Default: delegates to putCellCidsAndProofs batch. + ## + + await self.putCellCidsAndProofs(treeCid, @[(index, cellCid, blkCid, proof)]) + +########################################################### +# Delete methods +########################################################### method delBlock*( self: BlockStore, cid: Cid ): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Delete a block from the blockstore + ## Delete a block from the blockstore by CID ## raiseAssert("delBlock not implemented!") +method delBlocks*( + self: BlockStore, treeCid: Cid, indices: seq[Natural] +): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} = + ## Delete multiple blocks by tree CID and indices as a batch + ## + + raiseAssert("delBlocks by treeCid not implemented!") + method delBlock*( self: BlockStore, treeCid: Cid, index: Natural ): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Delete a block from the blockstore + ## Delete a block by tree CID and index. + ## Default: delegates to delBlocks batch. ## - raiseAssert("delBlock not implemented!") + await self.delBlocks(treeCid, @[index]) + +########################################################### +# Has methods +########################################################### method hasBlock*( self: BlockStore, cid: Cid @@ -144,13 +268,45 @@ method hasBlock*( raiseAssert("hasBlock not implemented!") +method hasBlocks*( + self: BlockStore, tree: Cid, indices: seq[Natural] +): Future[?!seq[(Natural, bool)]] {.base, async: (raises: [CancelledError]), gcsafe.} = + ## Check if multiple blocks exist in the blockstore + ## + + raiseAssert("hasBlocks not implemented!") + method hasBlock*( self: BlockStore, tree: Cid, index: Natural ): Future[?!bool] {.base, async: (raises: [CancelledError]), gcsafe.} = - ## Check if the block exists in the blockstore + ## Check if block exists by tree CID and index. + ## Default: delegates to hasBlocks batch. ## - raiseAssert("hasBlock not implemented!") + let results = ?await self.hasBlocks(tree, @[index]) + if results.len == 0: + return success(false) + success(results[0][1]) + +########################################################### +# Lifecycle / special methods +########################################################### + +method storeManifest*( + self: BlockStore, manifest: Manifest +): Future[?!Block] {.base, async: (raises: [CancelledError]), gcsafe.} = + ## Store a manifest to the blockstore and return the manifest block + ## + + raiseAssert("storeManifest not implemented!") + +method fetchManifest*( + self: BlockStore, cid: Cid +): Future[?!Manifest] {.base, async: (raises: [CancelledError]), gcsafe.} = + ## Fetch a manifest from the blockstore by CID + ## + + raiseAssert("fetchManifest not implemented!") method listBlocks*( self: BlockStore, blockType = BlockType.Manifest @@ -167,6 +323,10 @@ method close*(self: BlockStore): Future[void] {.base, async: (raises: []), gcsaf raiseAssert("close not implemented!") +########################################################### +# Convenience helpers +########################################################### + proc contains*( self: BlockStore, blk: Cid ): Future[bool] {.async: (raises: [CancelledError]), gcsafe.} = @@ -177,10 +337,6 @@ proc contains*( return (await self.hasBlock(blk)) |? false proc contains*( - self: BlockStore, address: BlockAddress + self: BlockStore, treeCid: Cid, index: Natural ): Future[bool] {.async: (raises: [CancelledError]), gcsafe.} = - return - if address.leaf: - (await self.hasBlock(address.treeCid, address.index)) |? false - else: - (await self.hasBlock(address.cid)) |? false + return (await self.hasBlock(treeCid, index)) |? false diff --git a/archivist/stores/cachestore.nim b/archivist/stores/cachestore.nim deleted file mode 100644 index 9c5b2136..00000000 --- a/archivist/stores/cachestore.nim +++ /dev/null @@ -1,306 +0,0 @@ -## Copyright (c) 2025 Archivist Authors -## Copyright (c) 2021 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [].} - -import std/options - -import pkg/chronos -import pkg/libp2p -import pkg/lrucache -import pkg/questionable -import pkg/questionable/results - -import ./blockstore -import ../units -import ../chunker -import ../errors -import ../logutils -import ../manifest -import ../merkletree -import ../utils -import ../clock - -export blockstore - -logScope: - topics = "archivist cachestore" - -type - CacheStore* = ref object of BlockStore - currentSize*: NBytes - size*: NBytes - cache: LruCache[Cid, Block] - cidAndProofCache: LruCache[(Cid, Natural), (Cid, ArchivistProof)] - - InvalidBlockSize* = object of ArchivistError - -const DefaultCacheSize*: NBytes = 5.MiBs - -method getBlock*( - self: CacheStore, cid: Cid -): Future[?!Block] {.async: (raises: [CancelledError]).} = - ## Get a block from the stores - ## - - trace "Getting block from cache", cid - - if cid.isEmpty: - trace "Empty block, ignoring" - return cid.emptyBlock - - if cid notin self.cache: - return failure (ref BlockNotFoundError)(msg: "Block not in cache " & $cid) - - try: - return success self.cache[cid] - except CancelledError as error: - raise error - except CatchableError as exc: - trace "Error requesting block from cache", cid, error = exc.msg - return failure exc - -method getCidAndProof*( - self: CacheStore, treeCid: Cid, index: Natural -): Future[?!(Cid, ArchivistProof)] {.async: (raises: [CancelledError]).} = - if cidAndProof =? self.cidAndProofCache.getOption((treeCid, index)): - success(cidAndProof) - else: - failure( - newException( - BlockNotFoundError, "Block not in cache: " & $BlockAddress.init(treeCid, index) - ) - ) - -method getBlock*( - self: CacheStore, treeCid: Cid, index: Natural -): Future[?!Block] {.async: (raises: [CancelledError]).} = - without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err: - return failure(err) - - await self.getBlock(cidAndProof[0]) - -method getBlockAndProof*( - self: CacheStore, treeCid: Cid, index: Natural -): Future[?!(Block, ArchivistProof)] {.async: (raises: [CancelledError]).} = - without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err: - return failure(err) - - let (cid, proof) = cidAndProof - - without blk =? await self.getBlock(cid), err: - return failure(err) - - success((blk, proof)) - -method getBlock*( - self: CacheStore, address: BlockAddress -): Future[?!Block] {.async: (raw: true, raises: [CancelledError]).} = - if address.leaf: - self.getBlock(address.treeCid, address.index) - else: - self.getBlock(address.cid) - -method hasBlock*( - self: CacheStore, cid: Cid -): Future[?!bool] {.async: (raises: [CancelledError]).} = - ## Check if the block exists in the blockstore - ## - - trace "Checking CacheStore for block presence", cid - if cid.isEmpty: - trace "Empty block, ignoring" - return true.success - - return (cid in self.cache).success - -method hasBlock*( - self: CacheStore, treeCid: Cid, index: Natural -): Future[?!bool] {.async: (raises: [CancelledError]).} = - without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err: - if err of BlockNotFoundError: - return success(false) - else: - return failure(err) - - await self.hasBlock(cidAndProof[0]) - -func cids(self: CacheStore): (iterator (): Cid {.gcsafe, raises: [].}) = - return - iterator (): Cid = - for cid in self.cache.keys: - yield cid - -method listBlocks*( - self: CacheStore, blockType = BlockType.Manifest -): Future[?!SafeAsyncIter[Cid]] {.async: (raises: [CancelledError]).} = - ## Get the list of blocks in the BlockStore. This is an intensive operation - ## - - let cids = self.cids() - - proc isFinished(): bool = - return finished(cids) - - proc genNext(): Future[?!Cid] {.async: (raises: [CancelledError]).} = - success(cids()) - - let iter = await ( - SafeAsyncIter[Cid].new(genNext, isFinished).filter( - proc(cid: ?!Cid): Future[bool] {.async: (raises: [CancelledError]).} = - without cid =? cid, err: - trace "Cannot get Cid from the iterator", err = err.msg - return false - without isManifest =? cid.isManifest, err: - trace "Error checking if cid is a manifest", err = err.msg - return false - - case blockType - of BlockType.Both: - return true - of BlockType.Manifest: - return isManifest - of BlockType.Block: - return not isManifest - ) - ) - success(iter) - -func putBlockSync(self: CacheStore, blk: Block): bool = - let blkSize = blk.data.len.NBytes # in bytes - - if blkSize > self.size: - trace "Block size is larger than cache size", blk = blkSize, cache = self.size - return false - - while self.currentSize + blkSize > self.size: - try: - let removed = self.cache.removeLru() - self.currentSize -= removed.data.len.NBytes - except EmptyLruCacheError as exc: - # if the cache is empty, can't remove anything, so break and add item - # to the cache - trace "Exception puting block to cache", exc = exc.msg - break - - self.cache[blk.cid] = blk - self.currentSize += blkSize - return true - -method putBlock*( - self: CacheStore, blk: Block, ttl = Duration.none -): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Put a block to the blockstore - ## - - trace "Storing block in cache", cid = blk.cid - if blk.isEmpty: - trace "Empty block, ignoring" - return success() - - discard self.putBlockSync(blk) - if onBlock =? self.onBlockStored: - await onBlock(blk.cid) - - return success() - -method putCidAndProof*( - self: CacheStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: ArchivistProof -): Future[?!void] {.async: (raises: [CancelledError]).} = - self.cidAndProofCache[(treeCid, index)] = (blockCid, proof) - success() - -method ensureExpiry*( - self: CacheStore, cid: Cid, expiry: SecondsSince1970 -): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Updates block's assosicated TTL in store - not applicable for CacheStore - ## - - discard # CacheStore does not have notion of TTL - -method ensureExpiry*( - self: CacheStore, treeCid: Cid, index: Natural, expiry: SecondsSince1970 -): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Updates block's associated TTL in store - not applicable for CacheStore - ## - - discard # CacheStore does not have notion of TTL - -method delBlock*( - self: CacheStore, cid: Cid -): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Delete a block from the blockstore - ## - - trace "Deleting block from cache", cid - if cid.isEmpty: - trace "Empty block, ignoring" - return success() - - let removed = self.cache.del(cid) - if removed.isSome: - self.currentSize -= removed.get.data.len.NBytes - - return success() - -method delBlock*( - self: CacheStore, treeCid: Cid, index: Natural -): Future[?!void] {.async: (raises: [CancelledError]).} = - let maybeRemoved = self.cidAndProofCache.del((treeCid, index)) - - if removed =? maybeRemoved: - return await self.delBlock(removed[0]) - - return success() - -method completeBlock*(self: CacheStore, address: BlockAddress, blk: Block) {.gcsafe.} = - discard - -method close*(self: CacheStore): Future[void] {.async: (raises: []).} = - ## Close the blockstore, a no-op for this implementation - ## - - discard - -proc new*( - _: type CacheStore, - blocks: openArray[Block] = [], - cacheSize: NBytes = DefaultCacheSize, - chunkSize: NBytes = DefaultChunkSize, -): CacheStore {.raises: [Defect, ValueError].} = - ## Create a new CacheStore instance - ## - ## `cacheSize` and `chunkSize` are both in bytes - ## - - if cacheSize < chunkSize: - raise newException(ValueError, "cacheSize cannot be less than chunkSize") - - let - currentSize = 0'nb - size = int(cacheSize div chunkSize) - cache = newLruCache[Cid, Block](size) - cidAndProofCache = newLruCache[(Cid, Natural), (Cid, ArchivistProof)](size) - store = CacheStore( - cache: cache, - cidAndProofCache: cidAndProofCache, - currentSize: currentSize, - size: cacheSize, - onBlockStored: CidCallback.none, - ) - - for blk in blocks: - discard store.putBlockSync(blk) - - return store - -proc new*( - _: type CacheStore, blocks: openArray[Block] = [], cacheSize: int, chunkSize: int -): CacheStore {.raises: [Defect, ValueError].} = - CacheStore.new(blocks, NBytes cacheSize, NBytes chunkSize) diff --git a/archivist/stores/keyutils.nim b/archivist/stores/keyutils.nim index e2f575f8..d0a357fe 100644 --- a/archivist/stores/keyutils.nim +++ b/archivist/stores/keyutils.nim @@ -7,9 +7,8 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import std/sugar import pkg/questionable/results -import pkg/datastore +import pkg/kvstore import pkg/libp2p import ../namespaces import ../manifest @@ -22,26 +21,42 @@ const ArchivistBlocksKey* = Key.init(ArchivistBlocksNamespace).tryGet ArchivistTotalBlocksKey* = Key.init(ArchivistBlockTotalNamespace).tryGet ArchivistManifestKey* = Key.init(ArchivistManifestNamespace).tryGet - BlocksTtlKey* = Key.init(ArchivistBlocksTtlNamespace).tryGet - BlockProofKey* = Key.init(ArchivistBlockProofNamespace).tryGet + ArchivistOverlaysKey* = Key.init(ArchivistOverlayNamespace).tryGet + BlocksMetaKey* = Key.init(ArchivistBlocksMetaNamespace).tryGet + BlockLeafKey* = Key.init(ArchivistBlockLeafNamespace).tryGet QuotaKey* = Key.init(ArchivistQuotaNamespace).tryGet QuotaUsedKey* = (QuotaKey / "used").tryGet QuotaReservedKey* = (QuotaKey / "reserved").tryGet -func makePrefixKey*(postFixLen: int, cid: Cid): ?!Key = - let cidKey = ?Key.init(($cid)[^postFixLen ..^ 1] & "/" & $cid) - +func makePrefixKey*(postFixLen: int, cid: Cid): ?!Key {.inline.} = + let cidStr = $cid if ?cid.isManifest: - success ArchivistManifestKey / cidKey + Key.init( + ArchivistManifestNamespace & "/" & cidStr[^postFixLen ..^ 1] & "/" & cidStr + ) else: - success ArchivistBlocksKey / cidKey + Key.init(ArchivistBlocksNamespace & "/" & cidStr[^postFixLen ..^ 1] & "/" & cidStr) + +func overlayKey*(treeCid: Cid): ?!Key = + ## Key for dataset overlay metadata: /meta/datasets/{treeCid} + ArchivistOverlaysKey / $treeCid + +func overlayQueryKey*(): ?!Key = + ## Query key for iterating all datasets: /meta/datasets/* + Key.init(?(ArchivistOverlaysKey / "*")) + +func blockMetaKey*(cid: Cid): ?!Key {.inline.} = + Key.init(ArchivistBlocksMetaNamespace & "/" & $cid) + +proc blockMetaKeyQuery*(): ?!Key = + Key.init(?(BlocksMetaKey / "*")) -proc createBlockExpirationMetadataKey*(cid: Cid): ?!Key = - BlocksTtlKey / $cid +func blockLeafKey*(treeCid: Cid, index: Natural): ?!Key {.inline.} = + Key.init(ArchivistBlockLeafNamespace & "/" & $treeCid & "/" & $index) -proc createBlockExpirationMetadataQueryKey*(): ?!Key = - let queryString = ?(BlocksTtlKey / "*") - Key.init(queryString) +func blockLeafKey*(treeCidStr: string, index: Natural): ?!Key {.inline.} = + Key.init(ArchivistBlockLeafNamespace & "/" & treeCidStr & "/" & $index) -proc createBlockCidAndProofMetadataKey*(treeCid: Cid, index: Natural): ?!Key = - (BlockProofKey / $treeCid).flatMap((k: Key) => k / $index) +func blockLeafQueryKey*(treeCid: Cid): ?!Key = + ## Query key for iterating all leafs under a tree: /meta/leafs/{treeCid}/* + Key.init(?(BlockLeafKey / $treeCid / "*")) diff --git a/archivist/stores/maintenance.nim b/archivist/stores/maintenance.nim index 2841298f..39607de6 100644 --- a/archivist/stores/maintenance.nim +++ b/archivist/stores/maintenance.nim @@ -8,7 +8,7 @@ ## those terms. ## Store maintenance module -## Looks for and removes expired blocks from blockstores. +## Scans overlays for expiration and drops expired ones. {.push raises: [].} @@ -35,8 +35,6 @@ type BlockMaintainer* = ref object of RootObj interval: Duration timer: Timer clock: Clock - numberOfBlocksPerInterval: int - offset: int proc new*( T: type BlockMaintainer, @@ -48,64 +46,57 @@ proc new*( ): BlockMaintainer = ## Create new BlockMaintainer instance ## - ## Call `start` to begin looking for for expired blocks + ## Call `start` to begin scanning for expired overlays ## - BlockMaintainer( - repoStore: repoStore, - interval: interval, - numberOfBlocksPerInterval: numberOfBlocksPerInterval, - timer: timer, - clock: clock, - offset: 0, - ) - -proc deleteExpiredBlock( - self: BlockMaintainer, cid: Cid -): Future[void] {.async: (raises: [CancelledError]).} = - if error =? (await self.repoStore.delBlock(cid)).errorOption: - warn "Unable to delete block from repoStore", error = error.msg - -proc processBlockExpiration( - self: BlockMaintainer, be: BlockExpiration -): Future[void] {.async: (raises: [CancelledError]).} = - if be.expiry < self.clock.now: - await self.deleteExpiredBlock(be.cid) - else: - inc self.offset + BlockMaintainer(repoStore: repoStore, interval: interval, timer: timer, clock: clock) -proc runBlockCheck( +proc dropExpiredOverlays( self: BlockMaintainer ): Future[void] {.async: (raises: [CancelledError]).} = - let expirations = await self.repoStore.getBlockExpirations( - maxNumber = self.numberOfBlocksPerInterval, offset = self.offset - ) - - without iter =? expirations, err: - warn "Unable to obtain blockExpirations iterator from repoStore", err = err.msg + without iter =? (await self.repoStore.listOverlays()), err: + warn "Unable to list overlays", err = err.msg return - var numberReceived = 0 - for beFut in iter: - without be =? (await beFut), err: - warn "Unable to obtain blockExpiration from iterator", err = err.msg + defer: + if err =? (await iter.dispose()).errorOption: + warn "Error disposing overlay iterator", err = err.msg + + let now = self.clock.now + + for cidFut in iter: + without treeCid =? (await cidFut), err: + warn "Unable to get overlay CID from iterator", err = err.msg continue - inc numberReceived - await self.processBlockExpiration(be) - await sleepAsync(1.millis) # cooperative scheduling - # If we received fewer blockExpirations from the iterator than we asked for, - # We're at the end of the dataset and should start from 0 next time. - if numberReceived < self.numberOfBlocksPerInterval: - self.offset = 0 - trace "Cycle completed" + without meta =? (await self.repoStore.getOverlay(treeCid)), err: + warn "Unable to get overlay metadata", treeCid, err = err.msg + continue + + # Deleting - finish cleanup, if delete in progress, dropOverlay will + # no-op + # Failure - always drop + # Any other status - check expiry + let shouldDrop = + meta.status == Deleting or meta.status == Failure or + (meta.expiry > 0 and meta.expiry < now) + + if shouldDrop: + trace "Dropping overlay", treeCid, status = meta.status, expiry = meta.expiry + if err =? (await self.repoStore.dropOverlay(treeCid)).errorOption: + error "Error dropping overlay", treeCid, status = meta.status, err = err.msg + + if manifestCid =? meta.manifestCid: + if err =? (await self.repoStore.delBlock(manifestCid)).errorOption: + warn "Error dropping manifest", err = err.msg + + await sleepAsync(1.millis) # cooperative scheduling proc start*(self: BlockMaintainer) = proc onTimer(): Future[void] {.async: (raises: []).} = try: - await self.runBlockCheck() + await self.dropExpiredOverlays() except CancelledError as err: - trace "Running block check in block maintenance timer callback cancelled: ", - err = err.msg + trace "Maintenance timer callback cancelled", err = err.msg self.timer.start(onTimer, self.interval) diff --git a/archivist/stores/networkstore.nim b/archivist/stores/networkstore.nim index 1a76c2eb..c1c4e034 100644 --- a/archivist/stores/networkstore.nim +++ b/archivist/stores/networkstore.nim @@ -9,14 +9,17 @@ {.push raises: [].} +import std/sequtils +import std/sets + import pkg/chronos import pkg/libp2p import pkg/questionable/results -import ../clock import ../blocktype import ../blockexchange import ../logutils +import ../manifest import ../merkletree import ../utils/asyncheapqueue import ../utils/safeasynciter @@ -31,16 +34,62 @@ type NetworkStore* = ref object of BlockStore engine*: BlockExcEngine # blockexc decision engine localStore*: BlockStore # local block store +method getBlocks*( + self: NetworkStore, cids: seq[Cid] +): Future[?!seq[Block]] {.async: (raises: [CancelledError]).} = + ## Get multiple blocks by CID (no tree context). + ## + ## Fetches all locally available blocks in one batch call. + ## For any missing blocks, falls back to individual network requests + ## (concurrent, one per missing CID). + ## + + let + uniqueCids = cids.toHashSet.toSeq + localBlocks = ?await self.localStore.getBlocks(uniqueCids) + + if localBlocks.len == uniqueCids.len: + return success(localBlocks) + + # Find CIDs not returned locally + let localCids = localBlocks.mapIt(it.cid).toHashSet + var + toRequestCids = (uniqueCids.toHashSet - localCids).toSeq + requests: seq[Future[?!Block]] + + for cid in toRequestCids: + requests.add(self.engine.requestBlock(BlockAddress.init(cid))) + + var allBlocks = localBlocks + while requests.len > 0: + without completedFut =? catchAsync(await one(requests)), err: + error "Unable to get block from exchange engine", err = err.msg + break + + let idx = requests.find(completedFut) + requests.del(idx) + + without blk =? catchAsync(await completedFut).flatten, err: + error "Unable to get block from exchange engine", err = err.msg + continue + + allBlocks.add(blk) + + success(allBlocks) + method getBlock*( - self: NetworkStore, address: BlockAddress + self: NetworkStore, cid: Cid ): Future[?!Block] {.async: (raises: [CancelledError]).} = - without blk =? (await self.localStore.getBlock(address)), err: + ## Get a block from the blockstore + ## + + without blk =? (await self.localStore.getBlock(cid)), err: if not (err of BlockNotFoundError): - error "Error getting block from local store", address, err = err.msg + error "Error getting block from local store", cid, err = err.msg return failure err - without newBlock =? (await self.engine.requestBlock(address)), err: - error "Unable to get block from exchange engine", address, err = err.msg + without newBlock =? (await self.engine.requestBlock(BlockAddress.init(cid))), err: + error "Unable to get block from exchange engine", cid, err = err.msg return failure err return success newBlock @@ -48,23 +97,75 @@ method getBlock*( return success blk method getBlock*( - self: NetworkStore, cid: Cid -): Future[?!Block] {.async: (raw: true, raises: [CancelledError]).} = + self: NetworkStore, treeCid: Cid, index: Natural +): Future[?!Block] {.async: (raises: [CancelledError]).} = ## Get a block from the blockstore ## - self.getBlock(BlockAddress.init(cid)) + without blk =? (await self.localStore.getBlock(treeCid, index)), err: + if not (err of BlockNotFoundError): + error "Error getting block from local store", treeCid, index, err = err.msg + return failure err -method getBlock*( - self: NetworkStore, treeCid: Cid, index: Natural -): Future[?!Block] {.async: (raw: true, raises: [CancelledError]).} = - ## Get a block from the blockstore + without newBlock =? + (await self.engine.requestBlock(BlockAddress.init(treeCid, index))), err: + error "Unable to get block from exchange engine", treeCid, index, err = err.msg + return failure err + + return success newBlock + + return success blk + +method getBlocks*( + self: NetworkStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Natural, Block)]] {.async: (raises: [CancelledError]).} = + ## Get multiple blocks by tree CID and indices. + ## + ## Fetches all locally available blocks in one batch call. + ## For any missing blocks, falls back to individual network requests + ## (concurrent, one per missing index). ## - self.getBlock(BlockAddress.init(treeCid, index)) + let + indices = indices.toHashSet.toSeq + localBlocks = ?await self.localStore.getBlocks(treeCid, indices) + + trace "Got local blocks", count = localBlocks.len + + # If all indices returned, we're done + if localBlocks.len == indices.len: + return success(localBlocks) + + # check get the diff of the still to retrieve indices from the network + var + toRequestIdxs = (indices.toHashSet - localBlocks.mapIt(it[0]).toHashSet).toSeq + requests: seq[Future[?!Block]] + + for idx in toRequestIdxs: + requests.add(self.engine.requestBlock(BlockAddress.init(treeCid, idx))) + + var allBlocks = localBlocks + while requests.len > 0: + without completedFut =? catchAsync(await one(requests)), err: + error "Unable to get block from exchange engine", treeCid, err = err.msg + break + + let + idx = requests.find(completedFut) + originalIdx = toRequestIdxs[idx] + requests.del(idx) + toRequestIdxs.del(idx) + + without blk =? catchAsync(await completedFut).flatten, err: + error "Unable to get block from exchange engine", treeCid, err = err.msg + continue -method completeBlock*(self: NetworkStore, address: BlockAddress, blk: Block) = - self.engine.completeBlock(address, blk) + allBlocks.add((originalIdx, blk)) + + success allBlocks + +method completeBlock*(self: NetworkStore, treeCid: Cid, index: Natural, blk: Block) = + self.engine.completeBlock(BlockAddress.init(treeCid, index), blk) method putBlock*( self: NetworkStore, blk: Block, ttl = Duration.none @@ -78,56 +179,20 @@ method putBlock*( await self.engine.resolveBlocks(@[blk]) return success() -method putCidAndProof*( - self: NetworkStore, - treeCid: Cid, - index: Natural, - blockCid: Cid, - proof: ArchivistProof, -): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} = - self.localStore.putCidAndProof(treeCid, index, blockCid, proof) - -method getCidAndProof*( - self: NetworkStore, treeCid: Cid, index: Natural -): Future[?!(Cid, ArchivistProof)] {.async: (raw: true, raises: [CancelledError]).} = - ## Get a block proof from the blockstore - ## - - self.localStore.getCidAndProof(treeCid, index) - -method ensureExpiry*( - self: NetworkStore, cid: Cid, expiry: SecondsSince1970 +method putBlocks*( + self: NetworkStore, treeCid: Cid, items: seq[(Block, Natural, ArchivistProof)] ): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Ensure that block's assosicated expiry is at least given timestamp - ## If the current expiry is lower then it is updated to the given one, otherwise it is left intact + ## Store leafs and blocks locally and notify the network ## - without blockCheck =? await self.localStore.hasBlock(cid), err: - return failure(err) - - if blockCheck: - return await self.localStore.ensureExpiry(cid, expiry) - else: - trace "Updating expiry - block not in local store", cid - + ?await self.localStore.putBlocks(treeCid, items) + await self.engine.resolveBlocks(items.mapIt(it[0])) return success() -method ensureExpiry*( - self: NetworkStore, treeCid: Cid, index: Natural, expiry: SecondsSince1970 -): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Ensure that block's associated expiry is at least given timestamp - ## If the current expiry is lower then it is updated to the given one, otherwise it is left intact - ## - - without blockCheck =? await self.localStore.hasBlock(treeCid, index), err: - return failure(err) - - if blockCheck: - return await self.localStore.ensureExpiry(treeCid, index, expiry) - else: - trace "Updating expiry - block not in local store", treeCid, index - - return success() +method putCidsAndProofs*( + self: NetworkStore, treeCid: Cid, items: seq[(Natural, Cid, ArchivistProof)] +): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} = + self.localStore.putCidsAndProofs(treeCid, items) method listBlocks*( self: NetworkStore, blockType = BlockType.Manifest @@ -154,13 +219,142 @@ method hasBlock*( trace "Checking network store for block existence", cid return await self.localStore.hasBlock(cid) -method hasBlock*( - self: NetworkStore, tree: Cid, index: Natural -): Future[?!bool] {.async: (raises: [CancelledError]).} = - ## Check if the block exists in the blockstore +method hasBlocks*( + self: NetworkStore, tree: Cid, indices: seq[Natural] +): Future[?!seq[(Natural, bool)]] {.async: (raw: true, raises: [CancelledError]).} = + ## Check if multiple blocks exist in the blockstore + ## + + self.localStore.hasBlocks(tree, indices) + +method storeManifest*( + self: NetworkStore, manifest: Manifest +): Future[?!Block] {.async: (raw: true, raises: [CancelledError]).} = + ## Store a manifest to the blockstore + ## + + self.localStore.storeManifest(manifest) + +method fetchManifest*( + self: NetworkStore, cid: Cid +): Future[?!Manifest] {.async: (raises: [CancelledError]).} = + ## Fetch a manifest from the blockstore by CID + ## + + without manifest =? (await self.localStore.fetchManifest(cid)), err: + if err of BlockNotFoundError: + without manifestBlk =? (await self.engine.requestBlock(cid)), err: + error "Unable to fetch manifest block!", err = err.msg + return failure(err) + + return Manifest.decode(manifestBlk) + + return success manifest + +method getCid*( + self: NetworkStore, treeCid: Cid, index: Natural +): Future[?!Cid] {.async: (raw: true, raises: [CancelledError]).} = + ## Get a block CID given a tree and index + ## + + self.localStore.getCid(treeCid, index) + +method putCellCidsAndProofs*( + self: NetworkStore, treeCid: Cid, items: seq[(Natural, Cid, Cid, ArchivistProof)] +): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} = + ## Put multiple cell CIDs and proofs as a batch + ## + + self.localStore.putCellCidsAndProofs(treeCid, items) + +method delBlocks*( + self: NetworkStore, treeCid: Cid, indices: seq[Natural] +): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} = + ## Delete multiple blocks by tree CID and indices + ## + + self.localStore.delBlocks(treeCid, indices) + +method getBlocksAndProofs*( + self: NetworkStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Natural, Block, ArchivistProof)]] {.async: (raises: [CancelledError]).} = + ## Get multiple blocks and proofs. + ## + ## Fetches all locally available blocks in one batch call. + ## For any missing blocks, falls back to individual network requests + ## (concurrent, one per missing index). After the engine persists a + ## network-fetched block, we retrieve the proof from the local store. + ## + ## TODO: The engine should return both block and proof directly via a + ## future signaling mechanism, with the caller handling persistence. + ## Currently the engine persists internally, so we must re-query the + ## store for the proof after each network fetch. ## - trace "Checking network store for block existence", tree, index - return await self.localStore.hasBlock(tree, index) + + let + indices = indices.toHashSet.toSeq + localBlocks = ?await self.localStore.getBlocksAndProofs(treeCid, indices) + + trace "Got local blocks and proofs", count = localBlocks.len + + # If all indices returned, we're done + if localBlocks.len == indices.len: + return success(localBlocks) + + # Check the diff of the still to retrieve indices from the network + var + toRequestIdxs = (indices.toHashSet - localBlocks.mapIt(it[0]).toHashSet).toSeq + requests: seq[Future[?!Block]] + + for idx in toRequestIdxs: + requests.add(self.engine.requestBlock(BlockAddress.init(treeCid, idx))) + + var allBlocks = localBlocks + while requests.len > 0: + without completedFut =? catchAsync(await one(requests)), err: + error "Unable to get block from exchange engine", treeCid, err = err.msg + break + + let + idx = requests.find(completedFut) + originalIdx = toRequestIdxs[idx] + + requests.del(idx) + toRequestIdxs.del(idx) + + without blk =? catchAsync(await completedFut).flatten, err: + error "Unable to get block from exchange engine", treeCid, err = err.msg + continue + + # TODO: The engine should return (Block, ?Proof), but we haven't yet refactored that, + # however, it does persist it on disk, so we can fetch from the local store. + # Once the engine is properly refactored for batch requests and returning the correct + # combination of (Block, Proof), we'll need to change this. + let blockProof = ?await self.localStore.getBlocksAndProofs(treeCid, @[originalIdx]) + if blockProof.len == 0: + warn "Skipping block and proof, couldn't resolve", + treeCid, index = originalIdx, cid = blk.cid + continue + + allBlocks.add(blockProof[0]) + + success allBlocks + +method getCidsAndProofs*( + self: NetworkStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Cid, ArchivistProof)]] {.async: (raw: true, raises: [CancelledError]).} = + ## Get multiple CIDs and proofs + ## + + self.localStore.getCidsAndProofs(treeCid, indices) + +method putBlocks*( + self: NetworkStore, treeCid: Cid, blocks: seq[(Natural, Block)] +): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} = + ## Put multiple blocks without proofs + ## + + self.localStore.putBlocks(treeCid, blocks) method close*(self: NetworkStore): Future[void] {.async: (raises: []).} = ## Close the underlying local blockstore @@ -174,4 +368,5 @@ proc new*( ): NetworkStore = ## Create new instance of a NetworkStore ## + NetworkStore(localStore: localStore, engine: engine) diff --git a/archivist/stores/queryiterhelper.nim b/archivist/stores/queryiterhelper.nim index 61b5b530..81538394 100644 --- a/archivist/stores/queryiterhelper.nim +++ b/archivist/stores/queryiterhelper.nim @@ -2,7 +2,7 @@ import pkg/questionable import pkg/questionable/results import pkg/chronos import pkg/chronicles -import pkg/datastore/typedds +import pkg/kvstore import ../utils/asynciter import ../utils/safeasynciter @@ -11,64 +11,63 @@ import ../utils/safeasynciter type KeyVal*[T] = tuple[key: Key, value: T] -proc toSafeAsyncIter*[T](queryIter: QueryIter[T]): SafeAsyncIter[QueryResponse[T]] = - ## Converts `QueryIter[T]` to `SafeAsyncIter[QueryResponse[T]]` - - if queryIter.finished: - return SafeAsyncIter[QueryResponse[T]].empty() - - proc genNext(): Future[?!QueryResponse[T]] {.async: (raises: [CancelledError]).} = +proc toSafeAsyncIter*[T](queryIter: QueryIter[T]): SafeAsyncIter[?KVRecord[T]] = + ## Converts kvstore `QueryIter[T]` to `SafeAsyncIter[?KVRecord[T]]` + ## + proc genNext(): Future[?!(?KVRecord[T])] {.async: (raises: [CancelledError]).} = await queryIter.next() proc isFinished(): bool = queryIter.finished - SafeAsyncIter[QueryResponse[T]].new(genNext, isFinished) + proc onDispose(): Future[?!void] {.async: (raises: []).} = + # kvquery.dispose returns Future[?!void] - await and propagate errors + return await dispose(queryIter) + + proc isDisposed(): bool = + queryIter.disposed + + # Always create the wrapper - even if queryIter.finished, we still need + # to dispose the underlying iterator to release its resources + SafeAsyncIter[?KVRecord[T]].new( + genNext = genNext, + isFinished = isFinished, + dispose = onDispose, + isDisposed = isDisposed, + ) proc filterSuccess*[T]( - iter: AsyncIter[?!QueryResponse[T]] -): Future[AsyncIter[tuple[key: Key, value: T]]] {.async: (raises: [CancelledError]).} = + iter: AsyncIter[?!(?KVRecord[T])] +): Future[AsyncIter[KeyVal[T]]] {.async: (raises: [CancelledError]).} = ## Filters out any items that are not success - proc mapping(resOrErr: ?!QueryResponse[T]): Future[?KeyVal[T]] {.async.} = - without res =? resOrErr, error: - error "Error occurred when getting QueryResponse", msg = error.msg + proc mapping(resOrErr: ?!(?KVRecord[T])): Future[?KeyVal[T]] {.async.} = + without recordOpt =? resOrErr, error: + error "Error occurred when getting KVRecord", msg = error.msg return KeyVal[T].none - without key =? res.key: - warn "No key for a QueryResponse" + without record =? recordOpt: return KeyVal[T].none - without value =? res.value, error: - error "Error occurred when getting a value from QueryResponse", msg = error.msg - return KeyVal[T].none - - (key: key, value: value).some + (key: record.key, value: record.val).some - await mapFilter[?!QueryResponse[T], KeyVal[T]](iter, mapping) + await mapFilter[?!RecordOption[T], KeyVal[T]](iter, mapping) proc filterSuccess*[T]( - iter: SafeAsyncIter[QueryResponse[T]] -): Future[SafeAsyncIter[tuple[key: Key, value: T]]] {. - async: (raises: [CancelledError]) -.} = + iter: SafeAsyncIter[(?KVRecord[T])] +): Future[SafeAsyncIter[KeyVal[T]]] {.async: (raises: [CancelledError]).} = ## Filters out any items that are not success proc mapping( - resOrErr: ?!QueryResponse[T] + resOrErr: ?!(?KVRecord[T]) ): Future[Option[?!KeyVal[T]]] {.async: (raises: [CancelledError]).} = - without res =? resOrErr, error: - error "Error occurred when getting QueryResponse", msg = error.msg - return Result[KeyVal[T], ref CatchableError].none - - without key =? res.key: - warn "No key for a QueryResponse" + without recordOpt =? resOrErr, error: + error "Error occurred when getting KVRecord", msg = error.msg return Result[KeyVal[T], ref CatchableError].none - without value =? res.value, error: - error "Error occurred when getting a value from QueryResponse", msg = error.msg + without record =? recordOpt: return Result[KeyVal[T], ref CatchableError].none - some(success((key: key, value: value))) + some(success((key: record.key, value: record.val))) - await mapFilter[QueryResponse[T], KeyVal[T]](iter, mapping) + await mapFilter[?KVRecord[T], KeyVal[T]](iter, mapping) diff --git a/archivist/stores/repostore/coders.nim b/archivist/stores/repostore/coders.nim index 3b9f90cf..c28d3eba 100644 --- a/archivist/stores/repostore/coders.nim +++ b/archivist/stores/repostore/coders.nim @@ -6,53 +6,135 @@ ## at your option. ## This file may not be copied, modified, or distributed except according to ## those terms. + +## Protobuf serialization for RepoStore metadata types +## +## ```protobuf +## message QuotaUsage { +## uint64 used = 1; # NBytes +## uint64 reserved = 2; # NBytes +## } +## +## message BlockMetadata { +## bytes cid = 1; # Cid bytes +## uint64 refCount = 2; # Natural +## } ## +## message LeafMetadata { +## uint32 deleted = 1; # bool as uint +## bytes blkCid = 2; # Cid bytes +## bytes proof = 3; # ArchivistProof bytes (optional) +## bytes cellCid = 4; # the cid of the cell if isCell == true +## } +## ``` + +{.push raises: [].} import std/sugar -import pkg/libp2p/cid -import pkg/serde/json -import pkg/stew/byteutils +import pkg/libp2p/[cid, protobuf/minprotobuf] +import pkg/questionable/results import pkg/stew/endians2 import ./types import ../../errors import ../../merkletree -import ../../utils/json proc encode*(t: QuotaUsage): seq[byte] = - t.toJson().toBytes() + var pb = initProtoBuffer() + pb.write(1, t.used.uint64) + pb.write(2, t.reserved.uint64) + pb.finish() + pb.buffer -proc decode*(T: type QuotaUsage, bytes: seq[byte]): ?!T = - T.fromJson(bytes) +proc decode*(T: type QuotaUsage, bytes: openArray[byte]): ?!T = + var + pb = initProtoBuffer(bytes) + used: uint64 + reserved: uint64 -proc encode*(t: BlockMetadata): seq[byte] = - t.toJson().toBytes() + if pb.getField(1, used).isErr: + return failure("Unable to decode `used` from QuotaUsage") -proc decode*(T: type BlockMetadata, bytes: seq[byte]): ?!T = - T.fromJson(bytes) + if pb.getField(2, reserved).isErr: + return failure("Unable to decode `reserved` from QuotaUsage") -proc encode*(t: LeafMetadata): seq[byte] = - t.toJson().toBytes() + success QuotaUsage(used: used.NBytes, reserved: reserved.NBytes) -proc decode*(T: type LeafMetadata, bytes: seq[byte]): ?!T = - T.fromJson(bytes) +proc encode*(t: BlockMetadata): seq[byte] = + var pb = initProtoBuffer() + pb.write(1, t.cid.data.buffer) + pb.write(2, t.refCount.uint64) + pb.finish() + pb.buffer + +proc decode*(T: type BlockMetadata, bytes: openArray[byte]): ?!T = + var + pb = initProtoBuffer(bytes) + cidBytes: seq[byte] + refCount: uint64 + + if pb.getField(1, cidBytes).isErr: + return failure("Unable to decode `cid` from BlockMetadata") -proc encode*(t: DeleteResult): seq[byte] = - t.toJson().toBytes() + if pb.getField(2, refCount).isErr: + return failure("Unable to decode `refCount` from BlockMetadata") -proc decode*(T: type DeleteResult, bytes: seq[byte]): ?!T = - T.fromJson(bytes) + let blkCid = ?Cid.init(cidBytes).mapFailure -proc encode*(t: StoreResult): seq[byte] = - t.toJson().toBytes() + success BlockMetadata(cid: blkCid, refCount: refCount.Natural) -proc decode*(T: type StoreResult, bytes: seq[byte]): ?!T = - T.fromJson(bytes) +proc encode*(t: LeafMetadata): seq[byte] = + var pb = initProtoBuffer() + pb.write(1, t.deleted.uint32) + pb.write(2, t.blkCid.data.buffer) + + let proofBytes = t.proof.encode() + if proofBytes.len > 0: + pb.write(3, proofBytes) + + if t.isCell: + pb.write(4, t.cellCid.data.buffer) + + pb.finish() + pb.buffer + +proc decode*(T: type LeafMetadata, bytes: openArray[byte]): ?!T = + var + pb = initProtoBuffer(bytes) + deleted: uint32 + blkCidBytes: seq[byte] + proofBytes: seq[byte] + cellCidBytes: seq[byte] + + if pb.getField(1, deleted).isErr: + return failure("Unable to decode `deleted` from LeafMetadata") + + if pb.getField(2, blkCidBytes).isErr: + return failure("Unable to decode `blkCid` from LeafMetadata") + + discard pb.getField(3, proofBytes) # Optional field + discard pb.getField(4, cellCidBytes) # Optional field (cell leaves only) + + let + blkCid = ?Cid.init(blkCidBytes).mapFailure + proof = ?ArchivistProof.decode(proofBytes) + + if cellCidBytes.len > 0: + let cellCid = ?Cid.init(cellCidBytes).mapFailure + success LeafMetadata( + deleted: deleted.bool, + blkCid: blkCid, + proof: proof, + isCell: true, + cellCid: cellCid, + ) + else: + success LeafMetadata(deleted: deleted.bool, blkCid: blkCid, proof: proof) proc encode*(i: uint64): seq[byte] = @(i.toBytesBE) -proc decode*(T: type uint64, bytes: seq[byte]): ?!T = +proc decode*(T: type uint64, bytes: openArray[byte]): ?!T = if bytes.len >= sizeof(uint64): success(uint64.fromBytesBE(bytes)) else: @@ -61,5 +143,9 @@ proc decode*(T: type uint64, bytes: seq[byte]): ?!T = proc encode*(i: Natural | enum): seq[byte] = cast[uint64](i).encode -proc decode*(T: typedesc[Natural | enum], bytes: seq[byte]): ?!T = - uint64.decode(bytes).map((ui: uint64) => cast[T](ui)) +proc decode*(T: typedesc[Natural | enum], bytes: openArray[byte]): ?!T = + let ui = ?uint64.decode(bytes) + when T is enum: + if ui > T.high.uint64: + return failure("Invalid enum value " & $ui & " for " & $T) + success cast[T](ui) diff --git a/archivist/stores/repostore/operations.nim b/archivist/stores/repostore/operations.nim index 5b5a5fc8..8635ec89 100644 --- a/archivist/stores/repostore/operations.nim +++ b/archivist/stores/repostore/operations.nim @@ -1,29 +1,26 @@ -## Copyright (c) 2025 Archivist Authors -## Copyright (c) 2024 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. +{.push raises: [].} + +import std/sets +import std/tables import pkg/chronos -import pkg/chronos/futures -import pkg/datastore -import pkg/datastore/typedds +import pkg/kvstore import pkg/libp2p/cid import pkg/metrics +import pkg/stew/bitseqs import pkg/questionable import pkg/questionable/results import ./coders import ./types +import ./overlays/coders import ../blockstore import ../keyutils import ../../blocktype import ../../clock import ../../logutils import ../../merkletree +import ../../utils logScope: topics = "archivist repostore" @@ -32,227 +29,640 @@ declareGauge(archivist_repostore_blocks, "archivist repostore blocks") declareGauge(archivist_repostore_bytes_used, "archivist repostore bytes used") declareGauge(archivist_repostore_bytes_reserved, "archivist repostore bytes reserved") -proc putLeafMetadata*( - self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: ArchivistProof -): Future[?!StoreResultKind] {.async: (raises: [CancelledError]).} = - without key =? createBlockCidAndProofMetadataKey(treeCid, index), err: - return failure(err) - - await self.metaDs.modifyGet( - key, - proc( - maybeCurrMd: ?LeafMetadata - ): Future[(?LeafMetadata, StoreResultKind)] {.async.} = - var - md: LeafMetadata - res: StoreResultKind - - if currMd =? maybeCurrMd: - md = currMd - res = AlreadyInStore +## NOTE: It's very important that we understand the general flow and order of operations +## and their guarantees. +## +## We have two stores - metadata (metaDs) and blockstore (repoDs), backed by KVStores +## with potentialy different guarantees, this can lead to subtle bugs if we do not +## understand what those are. +## +## The KVStore provides CAS (compare-and-swap) semantics, as well as batched atomic +## operations (atomic*). However, the atomic operations are only available on the sqlite +## (and potentialy others in the future) backend, the FS backend, which is used for the +## on disk blocks, only has CAS semantics and it doesn't support atomic opperations +## (it will raise at runtime - parhaps we can make this compile time as well). By CAS +## semantics we mean that we won't update stale records, but it doesn't mean that a +## multikey updates will remain consistent, this is only guaranteed by atomic operations. +## +## The atomic operations preserve consistency even in the event of crashes, however it +## requires care with the order of operations. We should avoid writing blocks to the +## filesystem before writing the metadata, because that would require expensive filesystem +## scans (which we used to do) to find orphaned blocks. If we write the metadata first, +## we can always recover from missing on disk block, by either re-downloading or dropping +## the meta entry. +## +## For the metadata writes: +## - ALWAYS WRITE BOTH LEAFS AND BLOCK META (refCount) AS AN ATOMIC BATCH, and only after +## updating the metadata (both leafs and counters and anything else that requires consistency +## per block) write the block on disk. +## +## For the counter updates: +## - ALWAYS USE ATOMIC WRITES to avoid inconsistent updates. Only block store writes +## (i.e. writing the block to disk) should affect the blockcount and quota values, metadata +## should never touch those. +## + +proc updateCounters*( + self: RepoStore, quotaDelta = 0, reservedDelta = 0, blocksDelta = 0 +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Update counters + ## + + let updates = + @[ + KVRecord[QuotaUsage].init( + QuotaUsedKey, + QuotaUsage( + used: max(0, quotaDelta).NBytes, reserved: max(0, reservedDelta).NBytes + ), + ).toRaw, + KVRecord[Natural].init(ArchivistTotalBlocksKey, max(0, blocksDelta).Natural).toRaw, + ] + + # NOTE: We always attempt to insert with default values first, + # and rely on retries to perform the actual update + proc updateCountersMiddleware( + records: seq[RawKVRecord], conflicts: seq[Key] + ): Future[?!seq[RawKVRecord]] {.async: (raises: [CancelledError]), gcsafe.} = + var refreshed = (?await self.metaDs.get(conflicts)).mapIt((it.key, it)).toTable + + for record in toSeq(refreshed.values): + if record.key == QuotaUsedKey: + var + quotaRec = ?toRecord[QuotaUsage](record) + quotaUsed = max(0, (quotaRec.val.used.int + quotaDelta)).NBytes + quotaReserved = max(0, (quotaRec.val.reserved.int + reservedDelta)).NBytes + trace "Updating quota to", quotaUsed, quotaReserved + + quotaRec.val.used = quotaUsed + quotaRec.val.reserved = quotaReserved + refreshed[record.key] = quotaRec.toRaw + elif record.key == ArchivistTotalBlocksKey: + var + totalBlocksRec = ?toRecord[Natural](record) + totalBlocks = max(0, totalBlocksRec.val.int + blocksDelta).Natural + trace "Updating block count to", totalBlocks + + totalBlocksRec.val = totalBlocks + refreshed[record.key] = totalBlocksRec.toRaw else: - md = LeafMetadata(blkCid: blkCid, proof: proof) - res = Stored + return failure("Unrecongized key! " & $record.key) - (md.some, res), - ) + success toSeq(refreshed.values) -proc delLeafMetadata*( - self: RepoStore, treeCid: Cid, index: Natural -): Future[?!void] {.async: (raises: [CancelledError]).} = - without key =? createBlockCidAndProofMetadataKey(treeCid, index), err: - return failure(err) + trace "Updating counters", quotaDelta, reservedDelta, blocksDelta + ?await self.metaDs.tryPutAtomic(updates, maxRetries = 10, updateCountersMiddleware) - if err =? (await self.metaDs.delete(key)).errorOption: - return failure(err) + if quotaDelta != 0: + self.quotaUsage.used = max(0, self.quotaUsage.used.int + quotaDelta).NBytes + archivist_repostore_bytes_used.set(self.quotaUsage.used.int64) + + if reservedDelta != 0: + self.quotaUsage.reserved = + max(0, self.quotaUsage.reserved.int + reservedDelta).NBytes + archivist_repostore_bytes_reserved.set(self.quotaUsage.reserved.int64) + + if blocksDelta != 0: + self.totalBlocks = max(0, self.totalBlocks.int + blocksDelta).Natural + archivist_repostore_blocks.set(self.totalBlocks.int64) success() -proc getLeafMetadata*( - self: RepoStore, treeCid: Cid, index: Natural -): Future[?!LeafMetadata] {.async: (raises: [CancelledError]).} = - without key =? createBlockCidAndProofMetadataKey(treeCid, index), err: - return failure(err) +proc deleteBlocksMetaRecs*( + self: RepoStore, blocksMeta: seq[KVRecord[BlockMetadata]] +): Future[?!seq[KVRecord[BlockMetadata]]] {.async: (raises: [CancelledError]).} = + ## Delete blocks and metadata for refCount == 0 + ## + ## Return skipped Records or empty seq if none were skipped + ## + + trace "Deleting blocks metadata", count = blocksMeta.len + # delete meta keys + await self.metaDs.tryDelete( + blocksMeta.filterIt(it.val.refCount == 0), # never delete if refCount != 0 + maxRetries = 10, + proc( + records: seq[KVRecord[BlockMetadata]] + ): Future[?!seq[KVRecord[BlockMetadata]]] {.async: (raises: [CancelledError]).} = + let refreshed = (?await self.metaDs.get(records.mapIt(it.key), BlockMetadata)).filterIt( + it.val.refCount == 0 # never delete if refCount != 0 + ) + + trace "Refreshed record for block metadata", count = refreshed.len + success refreshed + , + ) - without leafMd =? await get[LeafMetadata](self.metaDs, key), err: - if err of DatastoreKeyNotFound: - return failure(newException(BlockNotFoundError, err.msg)) +proc delFromBlocksStore*( + self: RepoStore, cids: seq[Cid] +): Future[?!seq[Cid]] {.async: (raises: [CancelledError]).} = + ## Delete from local block store + ## + + trace "Deleting from block store", count = cids.len + let + deduped = cids.deduplicate().filterIt(not it.isEmpty) + keys = deduped.mapIt(?makePrefixKey(self.postFixLen, it)) + + # get fs blocks first, we need a valid KVRecord.token - + # (we should have a `drop` method in the kvstore, that + # bypases CAS for cituations like this) + toDelete = ?await self.repoDs.get(keys) + + # delete on disk blocks first - best effort if crash + # occurs we might miss some blocks, but we can recover + # the delete, since the metadata is still present + skipped = (?await self.repoDs.delete(toDelete.toKeyRecord)).toHashSet + + # Update counters + deletedCount = toDelete.len - skipped.len + + var deletedSize = 0 + for record in toDelete: + if record.key in skipped: + continue + + deletedSize += record.val.len + + ?await self.updateCounters(quotaDelta = -(deletedSize), blocksDelta = -(deletedCount)) + return success skipped.mapIt(?Cid.init(it.value).mapFailure) + +proc tryDeleteBlocks*( + self: RepoStore, cids: seq[Cid] +): Future[?!seq[Cid]] {.async: (raises: [CancelledError]).} = + ## We only delete the block if no metadata is present + ## OR the refCount is 0 + ## + ## Returns skipped cids or empty seq + ## + + trace "Deleting blocks", count = cids.len + + let dedupped = cids.deduplicate().filterIt(not it.isEmpty) + + # Check refcounts before deleting from disk - only delete blocks + # whose refCount is 0 (or that have no metadata at all) + var + toDelete: seq[Cid] + skippedByRefCount: seq[Cid] + + let blocksMeta = + ?await self.metaDs.get(dedupped.mapIt(?blockMetaKey(it)), BlockMetadata) + + var metaByKey = initTable[Key, KVRecord[BlockMetadata]]() + for rec in blocksMeta: + metaByKey[rec.key] = rec + + for cid in dedupped: + let key = ?blockMetaKey(cid) + if key in metaByKey: + let rec = ?catch(metaByKey[key]) + if rec.val.refCount == 0: + toDelete.add(cid) + else: + skippedByRefCount.add(cid) else: - return failure(err) + # No metadata - safe to delete + toDelete.add(cid) + + # Delete fs blocks for refCount == 0 only + let skippedCids = ?await self.delFromBlocksStore(toDelete) + + # Delete metadata for successfully deleted fs blocks + let deletedCids = toDelete.toHashSet - skippedCids.toHashSet + var deletedMetaKeys: HashSet[Key] + for cid in deletedCids: + deletedMetaKeys.incl(?blockMetaKey(cid)) + + var deletedMeta: seq[KVRecord[BlockMetadata]] + for rec in blocksMeta: + if rec.key in deletedMetaKeys: + deletedMeta.add(rec) + + discard ?await self.deleteBlocksMetaRecs(deletedMeta) - success(leafMd) + # Return all skipped cids (refCount > 0 + failed disk deletes) + success skippedByRefCount & skippedCids -proc updateTotalBlocksCount*( - self: RepoStore, plusCount: Natural = 0, minusCount: Natural = 0 +proc tryDeleteBlocks*( + self: RepoStore, cid: Cid +): Future[?!seq[Cid]] {.async: (raises: [CancelledError], raw: true).} = + self.tryDeleteBlocks(@[cid]) + +type BlockLeafTuple = + tuple[index: Natural, blkCid: Cid, cellCid: ?Cid, proof: ArchivistProof] + +proc putLeafBlockMetaImpl( + self: RepoStore, treeCid: Cid, blocks: seq[BlockLeafTuple] ): Future[?!void] {.async: (raises: [CancelledError]).} = - await self.metaDs.modify( - ArchivistTotalBlocksKey, - proc(maybeCurrCount: ?Natural): Future[?Natural] {.async.} = - let count: Natural = - if currCount =? maybeCurrCount: - currCount + plusCount - minusCount + ## Core implementation for leaf and block metadata writes. + ## + ## When the cellCid is set, we treat the leaf as a cell leaf, + ## and the index and treeCid corresponds to the cell index and + ## tree of the slot, when it isn't set, this is a regular block + ## leaf and it's index is the index of the leaf in a regular top + ## level tree. + ## + ## The main difference apart from representing different types of + ## objects, is that multiple cells point to the same block, + ## so they might increment the same block's refCount several times. + ## + ## All writes - leaf records, block refcounts, and overlay BitSeq - + ## happen in a single atomic transaction, so no partial state is ever + ## visible per batch. + ## + + # Fetch existing overlay to get correct BitSeq length + # (overlay must exist before putBlocks is called) + trace "Fetching existing overlay", treeCid = treeCid, blocksCount = blocks.len + + let + existingOverlayRec = ?await self.metaDs.get(?overlayKey(treeCid), OverlayMetadata) + treeCidStr = $treeCid + + var overlayMeta = existingOverlayRec.val + trace "Got existing overlay", treeCid, existingBitmapLen = overlayMeta.blocks.len + + # abort if overlay is already being deleted + if overlayMeta.status == Deleting: + return failure(newException(OverlayDeletingError, "Overlay is being deleted")) + + var + blkToLeafMap: Table[Key, (RawKVRecord, HashSet[RawKVRecord])] + leafsMap: Table[Key, RawKVRecord] + blocksBits = BitSeq.init(blocks.mapIt(it[0]).max() + 1) + + for (index, blkCid, cellCid, proof) in blocks: + let + blkKey = ?blockMetaKey(blkCid) + leafKey = ?blockLeafKey(treeCidStr, index) + + var + blkRec = + KVRecord[BlockMetadata].init(blkKey, BlockMetadata(refCount: 1, cid: blkCid)) + leafRec = + if cCid =? cellCid: + KVRecord[LeafMetadata].init( + leafKey, + LeafMetadata(blkCid: blkCid, proof: proof, isCell: true, cellCid: cCid), + ) else: - plusCount - minusCount + KVRecord[LeafMetadata].init( + leafKey, LeafMetadata(blkCid: blkCid, proof: proof) + ) + + # we only increase refcount for **NEW LEAVES**, if a leaf + # already exists, we skip the refCount.inc, thus we make + # a mapping of block rec -> leaf rec to be able to filter + # out inserts from updates + blocksBits.setBit(index) + leafsMap[leafKey] = leafRec.toRaw + + # Skip block metadata for empty blkCid (pad blocks) + if not blkCid.isEmpty: + blkToLeafMap.withValue(blkKey, pairs): + pairs[][1].incl(leafRec.toRaw) + blkRec.val.refCount = max(1, pairs[][1].len) + pairs[][0] = blkRec.toRaw + do: + blkToLeafMap[blkKey] = (blkRec.toRaw, [leafRec.toRaw].toHashSet) + + overlayMeta.blocks.combineSafe(blocksBits) + + proc putLeafAndBlockMetaAtomic( + records: seq[RawKVRecord], conflicts: seq[Key] + ): Future[?!seq[RawKVRecord]] {.async: (raises: [CancelledError]), gcsafe.} = + var + records = records.mapIt((it.key, it)).toTable + refreshed = ?await self.metaDs.get(conflicts) + + let conflictSet = conflicts.toHashSet + + trace "Got refreshed leaf and block records", + refreshed = refreshed.len, conflicts = conflicts.len + + # Update the overlay first, to avoid writing over a deleted overlays + for i, rec in refreshed: + if ArchivistOverlaysKey.ancestor(rec.key): + let overlayMetaRec = ?toRecord[OverlayMetadata](rec) + # Abort if overlay is being deleted + if overlayMetaRec.val.status == Deleting: + return failure(newException(OverlayDeletingError, "Overlay is being deleted")) + + # Update overlay and mark for removal + var updatedRec = overlayMetaRec + updatedRec.val.blocks.combineSafe(overlayMeta.blocks) + trace "Updated overlay meta", overlay = updatedRec.val + overlayMeta = updatedRec.val + records[rec.key] = updatedRec.toRaw + refreshed.del(i) + break + + for rec in refreshed: + var record = rec + + trace "Processing record", key = record.key + if BlockLeafKey.ancestor(record.key): + let incomingLeafRec = ?toRecord[LeafMetadata](?catch(leafsMap[record.key])) + var currentLeafRec = ?toRecord[LeafMetadata](record) + + currentLeafRec.val.deleted = incomingLeafRec.val.deleted + currentLeafRec.val.blkCid = incomingLeafRec.val.blkCid + + let + hasCurrentProof = + (not currentLeafRec.val.proof.isNil) and + currentLeafRec.val.proof.path.len > 0 + hasIncomingProof = + (not incomingLeafRec.val.proof.isNil) and + incomingLeafRec.val.proof.path.len > 0 + + if hasIncomingProof or not hasCurrentProof: + currentLeafRec.val.proof = incomingLeafRec.val.proof + + record = currentLeafRec.toRaw + elif BlocksMetaKey.ancestor(record.key): + var blockMeta = ?toRecord[BlockMetadata](record) + # Count only new leaf references (leaves NOT in conflict set) + blkToLeafMap.withValue(record.key, pairs): + let + (_, leafRecs) = pairs[] + + # get the intersection + newLeafs = (leafRecs.mapIt(it.key).toHashSet - conflictSet) + + if newLeafs.len > 0: + blockMeta.val.refCount += newLeafs.len.Natural + trace "Updated refCount for", + cid = blockMeta.val.cid, + refCount = blockMeta.val.refCount, + newLeafs = newLeafs.len + else: + trace "Skipping refCount increment (all leafs already existed)", + cid = blockMeta.val.cid, refCount = blockMeta.val.refCount + + record = blockMeta.toRaw + + # update records + records[record.key] = record + + trace "Records to put ", records = records.len + success toSeq(records.values) + + let updates = + @[existingOverlayRec.fromRecord(overlayMeta).toRaw] & + blkToLeafMap.values.toSeq.mapIt(it[0]) & leafsMap.values.toSeq + + trace "Put or update leaf and block metadata", treeCid, recordsCount = updates.len + if err =? ( + await self.metaDs.tryPutAtomic(updates, maxRetries = 10, putLeafAndBlockMetaAtomic) + ).errorOption: + trace "Unable to put or update leaf and block metadata", error = err.msg + return failure(err) + + # cache the final overlay + self.overlayCache[?overlayKey(treeCid)] = overlayMeta + + success() - self.totalBlocks = count - archivist_repostore_blocks.set(count.int64) - count.some, +proc putLeafBlockMeta*( + self: RepoStore, treeCid: Cid, blocks: seq[(Natural, Cid, ArchivistProof)] +): Future[?!void] {.async: (raises: [CancelledError], raw: true).} = + ## Put or update leaf and block metadata (plain blocks). + ## + self.putLeafBlockMetaImpl(treeCid, blocks.mapIt((it[0], it[1], Cid.none, it[2]))) + +proc putLeafBlockMeta*( + self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: ArchivistProof +): Future[?!void] {.async: (raises: [CancelledError], raw: true).} = + self.putLeafBlockMeta(treeCid, @[(index, blkCid, proof)]) + +proc putCellLeafBlockMeta*( + self: RepoStore, treeCid: Cid, blocks: seq[(Natural, Cid, Cid, ArchivistProof)] +): Future[?!void] {.async: (raises: [CancelledError], raw: true).} = + ## Put or update leaf and block metadata for slot proof cell leaves. + ## + ## Each item is (index, blkCid, blkCid, proof): + ## - cellCid: the cell digest (stored as LeafMetadata.cellCid) + ## - blkCid: the actual block CID (refcount key, LeafMetadata.blkCid) + ## + ## All updates (leaf record with cellCid, block refcount, overlay BitSeq) + ## are committed in a single atomic transaction. + ## + + self.putLeafBlockMetaImpl( + treeCid, + blocks.mapIt((index: it[0], blkCid: it[2], cellCid: it[1].some, proof: it[3])), ) -proc updateQuotaUsage*( - self: RepoStore, - plusUsed: NBytes = 0.NBytes, - minusUsed: NBytes = 0.NBytes, - plusReserved: NBytes = 0.NBytes, - minusReserved: NBytes = 0.NBytes, +proc delLeafBlockMetadata*( + self: RepoStore, treeCid: Cid, index: seq[Natural] ): Future[?!void] {.async: (raises: [CancelledError]).} = - await self.metaDs.modify( - QuotaUsedKey, - proc(maybeCurrUsage: ?QuotaUsage): Future[?QuotaUsage] {.async.} = - var usage: QuotaUsage - - if currUsage =? maybeCurrUsage: - usage = QuotaUsage( - used: currUsage.used + plusUsed - minusUsed, - reserved: currUsage.reserved + plusReserved - minusReserved, + ## Update leaf and block metadata, the delete is two step + ## to avoid refcount divergence: + ## + ## - We first do an atomic update of the block refcount and + ## set leafs as deleted = true. + ## - If a crash occurs in between, refCounts stay consistent + ## We then delete leafs and blocks who's refcount is 0. + ## + ## Optimized: First checks overlay BitSeq for fast-path rejection. + ## If none of the indices to delete have bits set, returns early. + ## + ## TODO: This is highly inefficient under the current schema. + ## To optimize this we need to avoid O(N) leaf -> block + ## scans (which end up being O(N^2), since we retrieve the same amount of + ## block metadata), we can do this by packing multiple leafs into a single + ## key (sharding the tree storage essentially), this becomes relevant as well + ## when we flatten the tree + + logScope: + treeCid = treeCid + + trace "Deleting leaf and block metadata" + + let + existingOverlayMeta = ?await self.metaDs.get(?overlayKey(treeCid), OverlayMetadata) + uniqueIdxs = index.deduplicate() + + var overlayMeta = existingOverlayMeta.val + if not uniqueIdxs.anyIt(it < overlayMeta.blocks.len and overlayMeta.blocks[it]): + trace "No bits set in BitSeq for indices to delete, fast-path return" + return success() + + # Continue with existing logic + let + treeCidStr = $treeCid + leafKeys = uniqueIdxs.mapIt(?blockLeafKey(treeCidStr, it)) + leafsMeta = + (?await self.metaDs.get(leafKeys, LeafMetadata)).filterIt(not it.val.deleted) + updateLeafsRecs = leafsMeta.mapIt( + it.fromRecord( + if it.val.isCell: + LeafMetadata( + blkCid: it.val.blkCid, + proof: it.val.proof, + deleted: true, + isCell: true, + cellCid: it.val.cellCid, + ) + else: + LeafMetadata(blkCid: it.val.blkCid, proof: it.val.proof, deleted: true) + ) + ) + + # Build aggregation table block key -> set of leaf indices + # This ensures we correctly decrement refCount when multiple leaves + # reference the same block + # Skip empty blkCid (pad blocks) - no block metadata to decrement + var blkToLeafIndices: Table[Key, HashSet[Natural]] + for leafMeta in leafsMeta: + if not leafMeta.val.blkCid.isEmpty: + let + blkKey = ?blockMetaKey(leafMeta.val.blkCid) + # Extract index from leaf key: /meta/leafs/{treeCid}/{index} + idx = ?catch(parseInt(leafMeta.key.value)) + + blkToLeafIndices.withValue(blkKey, indices): + indices[].incl(idx.Natural) + do: + blkToLeafIndices[blkKey] = [idx.Natural].toHashSet + + # Get unique block keys and build update records with correct refCount decrement + let + uniqueBlkKeys = toSeq(blkToLeafIndices.keys) + blksMeta = ?await self.metaDs.get(uniqueBlkKeys, BlockMetadata) + updateBlksRecs = blksMeta.mapIt( + block: + let leafCount = + blkToLeafIndices.getOrDefault(it.key, initHashSet[Natural]()).len + + it.fromRecord( + BlockMetadata(refCount: max(0, it.val.refCount - leafCount), cid: it.val.cid) ) + ) + + var blockBits = BitSeq.init(uniqueIdxs.max() + 1) + blockBits.combineSafe(overlayMeta.blocks) + for i in uniqueIdxs: + blockBits.clearBit(i) + + let deleteExpiry = self.clock.now() + overlayMeta.status = Deleting + overlayMeta.expiry = deleteExpiry + overlayMeta.blocks = blockBits + + proc atomicUpdateDelMeta( + records: seq[RawKVRecord], conflicts: seq[Key] + ): Future[?!seq[RawKVRecord]] {.async: (raises: [CancelledError]), gcsafe.} = + var records = records.mapIt((it.key, it)).toTable + + let refreshed = ?await self.metaDs.get(conflicts) + + trace "Got refreshed metadata", count = refreshed.len + for rec in refreshed: + var record = rec + trace "Processing record", key = record.key + if BlockLeafKey.ancestor(record.key): + var leaf = ?toRecord[LeafMetadata](record) + leaf.val.deleted = true # mark for delete + trace "Setting leaf to deleted", key = record.key + record = leaf.toRaw + elif BlocksMetaKey.ancestor(record.key): + var blkMeta = ?toRecord[BlockMetadata](record) + # Decrement by the count of leaves pointing to this block + let leafCount = + blkToLeafIndices.getOrDefault(record.key, initHashSet[Natural]()).len + trace "Before decrease refCount", + refCount = blkMeta.val.refCount, leafCount = leafCount + blkMeta.val.refCount = max(0, blkMeta.val.refCount - leafCount) + trace "Decreased refCount for block", + key = record.key, refCount = blkMeta.val.refCount + record = blkMeta.toRaw + elif ArchivistOverlaysKey.ancestor(record.key): + var overlayMetaRec = ?toRecord[OverlayMetadata](record) + # Re-apply clearBit operations on fresh overlay data + # to avoid clobbering concurrent updates + overlayMetaRec.val.blocks.combineSafe(overlayMeta.blocks) + for i in uniqueIdxs: + overlayMetaRec.val.blocks.clearBit(i) + + overlayMetaRec.val.status = Deleting + overlayMetaRec.val.expiry = deleteExpiry + + trace "Updated overlay meta for delete", overlay = overlayMetaRec.val + overlayMeta = overlayMetaRec.val + record = overlayMetaRec.toRaw else: - usage = - QuotaUsage(used: plusUsed - minusUsed, reserved: plusReserved - minusReserved) - - if usage.used + usage.reserved > self.quotaMaxBytes: - raise newException( - QuotaNotEnoughError, - "Quota usage would exceed the limit. Used: " & $usage.used & ", reserved: " & - $usage.reserved & ", limit: " & $self.quotaMaxBytes, + return failure( + "Got an unknown key updating leaf and block metadata - key: " & $record.key ) - else: - self.quotaUsage = usage - archivist_repostore_bytes_used.set(usage.used.int64) - archivist_repostore_bytes_reserved.set(usage.reserved.int64) - return usage.some, - ) -proc updateBlockMetadata*( - self: RepoStore, - cid: Cid, - plusRefCount: Natural = 0, - minusRefCount: Natural = 0, - minExpiry: SecondsSince1970 = 0, -): Future[?!void] {.async: (raises: [CancelledError]).} = - if cid.isEmpty: - return success() + # update records + records[record.key] = record - without metaKey =? createBlockExpirationMetadataKey(cid), err: - return failure(err) + trace "Refreshed leaf and block records", count = refreshed.len + success toSeq(records.values) - await self.metaDs.modify( - metaKey, - proc(maybeCurrBlockMd: ?BlockMetadata): Future[?BlockMetadata] {.async.} = - if currBlockMd =? maybeCurrBlockMd: - BlockMetadata( - size: currBlockMd.size, - expiry: max(currBlockMd.expiry, minExpiry), - refCount: currBlockMd.refCount + plusRefCount - minusRefCount, - ).some - else: - raise newException( - BlockNotFoundError, "Metadata for block with cid " & $cid & " not found" - ), + ?await self.metaDs.tryPutAtomic( + @[existingOverlayMeta.fromRecord(overlayMeta).toRaw] & + updateLeafsRecs.mapIt(it.toRaw) & updateBlksRecs.mapIt(it.toRaw), + maxRetries = 10, + atomicUpdateDelMeta, ) -proc storeBlock*( - self: RepoStore, blk: Block, minExpiry: SecondsSince1970 -): Future[?!StoreResult] {.async: (raises: [CancelledError]).} = - if blk.isEmpty: - return success(StoreResult(kind: AlreadyInStore)) + # cache the final overlay after delete + self.overlayCache[?overlayKey(treeCid)] = overlayMeta - without metaKey =? createBlockExpirationMetadataKey(blk.cid), err: - return failure(err) + let + toDeleteLeafMeta = + ?await self.metaDs.get(updateLeafsRecs.mapIt(it.key), LeafMetadata) + toDeleteBlockMeta = ( + ?await self.metaDs.get(updateBlksRecs.mapIt(it.key), BlockMetadata) + ).filterIt(it.val.refCount == 0) - without blkKey =? makePrefixKey(self.postFixLen, blk.cid), err: - return failure(err) + trace "Got leaf and block metadata", + leafMeta = toDeleteLeafMeta.len, blockMeta = toDeleteBlockMeta.len - await self.metaDs.modifyGet( - metaKey, - proc(maybeCurrMd: ?BlockMetadata): Future[(?BlockMetadata, StoreResult)] {.async.} = - var - md: BlockMetadata - res: StoreResult - - if currMd =? maybeCurrMd: - if currMd.size == blk.data.len.NBytes: - md = BlockMetadata( - size: currMd.size, - expiry: max(currMd.expiry, minExpiry), - refCount: currMd.refCount, - ) - res = StoreResult(kind: AlreadyInStore) + if toDeleteBlockMeta.len > 0: + let + skippedFs = + (?await self.delFromBlocksStore(toDeleteBlockMeta.mapIt(it.val.cid))).toHashSet - # making sure that the block actually is stored in the repoDs - without hasBlock =? await self.repoDs.has(blkKey), err: - raise err + skippedRecs = + ?await self.deleteBlocksMetaRecs( + toDeleteBlockMeta.filterIt(it.val.cid notin skippedFs) + ) - if not hasBlock: - warn "Block metadata is present, but block is absent. Restoring block.", - cid = blk.cid - if err =? (await self.repoDs.put(blkKey, blk.data)).errorOption: - raise err - else: - raise newException( - CatchableError, - "Repo already stores a block with the same cid but with a different size, cid: " & - $blk.cid, - ) - else: - md = BlockMetadata(size: blk.data.len.NBytes, expiry: minExpiry, refCount: 0) - res = StoreResult(kind: Stored, used: blk.data.len.NBytes) - if err =? (await self.repoDs.put(blkKey, blk.data)).errorOption: - raise err + if skippedRecs.len > 0: + trace "Some blocks were not deleted", skipped = skippedRecs.len - (md.some, res), - ) + if toDeleteLeafMeta.len > 0: + let failedDeletes = + # NOTE: actual deletes are optimistic, they will be picked up + # by the maintenance - blocks with refCount = 0 and leafs with + # delete = true are going to be dropped + ?await self.metaDs.delete(toDeleteLeafMeta) -proc tryDeleteBlock*( - self: RepoStore, cid: Cid, expiryLimit = SecondsSince1970.low -): Future[?!DeleteResult] {.async: (raises: [CancelledError]).} = - without metaKey =? createBlockExpirationMetadataKey(cid), err: - return failure(err) + if failedDeletes.len > 0: + trace "Some records failed to delete", failedDeletes = failedDeletes.len - without blkKey =? makePrefixKey(self.postFixLen, cid), err: - return failure(err) + success() - await self.metaDs.modifyGet( - metaKey, - proc( - maybeCurrMd: ?BlockMetadata - ): Future[(?BlockMetadata, DeleteResult)] {.async.} = - var - maybeMeta: ?BlockMetadata - res: DeleteResult - - if currMd =? maybeCurrMd: - if currMd.refCount == 0 or currMd.expiry < expiryLimit: - maybeMeta = BlockMetadata.none - res = DeleteResult(kind: Deleted, released: currMd.size) - - if err =? (await self.repoDs.delete(blkKey)).errorOption: - raise err - else: - maybeMeta = currMd.some - res = DeleteResult(kind: InUse) - else: - maybeMeta = BlockMetadata.none - res = DeleteResult(kind: NotFound) +proc delLeafBlockMetadata*( + self: RepoStore, treeCid: Cid, index: Natural +): Future[?!void] {.async: (raises: [CancelledError], raw: true).} = + self.delLeafBlockMetadata(treeCid, @[index]) - # making sure that the block acutally is removed from the repoDs - without hasBlock =? await self.repoDs.has(blkKey), err: - raise err +proc getLeafMetadata*( + self: RepoStore, treeCid: Cid, index: Natural +): Future[?!LeafMetadata] {.async: (raises: [CancelledError]).} = + let key = ?blockLeafKey(treeCid, index) - if hasBlock: - warn "Block metadata is absent, but block is present. Removing block.", cid - if err =? (await self.repoDs.delete(blkKey)).errorOption: - raise err + without leafMd =? await self.metaDs.get(key, LeafMetadata), err: + if err of KVStoreKeyNotFound: + return failure(newException(BlockNotFoundError, err.msg)) + else: + return failure(err) - (maybeMeta, res), - ) + success(leafMd.val) diff --git a/archivist/stores/repostore/overlays.nim b/archivist/stores/repostore/overlays.nim new file mode 100644 index 00000000..71c3242d --- /dev/null +++ b/archivist/stores/repostore/overlays.nim @@ -0,0 +1,4 @@ +import ./overlays/overlays +import ./overlays/coders + +export overlays, coders diff --git a/archivist/stores/repostore/overlays/coders.nim b/archivist/stores/repostore/overlays/coders.nim new file mode 100644 index 00000000..24dba600 --- /dev/null +++ b/archivist/stores/repostore/overlays/coders.nim @@ -0,0 +1,83 @@ +## Copyright (c) 2025 Archivist Authors +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +## Protobuf serialization for OverlayMetadata +## +## ```protobuf +## message OverlayMetadata { +## uint32 status = 1; # OverlayStatus enum +## uint64 expiry = 2; # SecondsSince1970 +## bytes blocks = 3; # Bitmap (bit N = block N present) +## bytes manifestCid = 4; # Cid bytes (optional, for cleanup) +## } +## ``` + +{.push raises: [].} + +import pkg/libp2p/[cid, protobuf/minprotobuf] +import pkg/questionable +import pkg/questionable/results +import pkg/stew/bitseqs + +import ../types +import ../../../errors + +proc encode*(meta: OverlayMetadata): seq[byte] = + ## Encode OverlayMetadata to protobuf bytes + ## + + var pb = initProtoBuffer() + + pb.write(1, meta.status.uint32) + pb.write(2, meta.expiry.uint64) + + let blocksBytes = seq[byte](meta.blocks) + if blocksBytes.len > 0: + pb.write(3, blocksBytes) + + if manifestCid =? meta.manifestCid: + pb.write(4, manifestCid.data.buffer) + + pb.finish() + pb.buffer + +proc decode*(T: type OverlayMetadata, data: openArray[byte]): ?!T = + ## Decode OverlayMetadata from protobuf bytes + ## + + var + pb = initProtoBuffer(data) + status: uint32 + expiry: uint64 + blocks: seq[byte] + manifestCidBytes: seq[byte] + + if pb.getField(1, status).isErr: + return failure("Unable to decode `status` from OverlayMetadata") + + if pb.getField(2, expiry).isErr: + return failure("Unable to decode `expiry` from OverlayMetadata") + + discard pb.getField(3, blocks) # Optional field + discard pb.getField(4, manifestCidBytes) # Optional field + + let manifestCid: ?Cid = + if manifestCidBytes.len > 0: + (?Cid.init(manifestCidBytes).mapFailure).some + else: + Cid.none + + if status.int > OverlayStatus.high.ord.int: + return failure("Invalid OverlayStatus value: " & $status) + + success OverlayMetadata( + status: OverlayStatus(status), + expiry: expiry.int64, + blocks: BitSeq blocks, + manifestCid: manifestCid, + ) diff --git a/archivist/stores/repostore/overlays/overlays.nim b/archivist/stores/repostore/overlays/overlays.nim new file mode 100644 index 00000000..4552504a --- /dev/null +++ b/archivist/stores/repostore/overlays/overlays.nim @@ -0,0 +1,468 @@ +## Overlay metadata operations - storing, retrieving, and listing dataset overlays + +{.push raises: [].} + +import std/algorithm +import std/strutils +import std/tables + +import pkg/chronos +import pkg/kvstore +import pkg/libp2p/[cid, multihash, multicodec] +import pkg/stew/bitseqs +import pkg/results +import pkg/questionable +import pkg/questionable/results + +import ./coders +import ../types +import ../operations +import ../../keyutils +import ../../../clock +import ../../../archivisttypes + +import ../../queryiterhelper + +import ../../../utils +import ../../../errors +import ../../../logutils +import ../../../rng + +export coders + +logScope: + topics = "archivist repostore overlays" + +proc mergeOverlay( + self: RepoStore, + overlay: var OverlayMetadata, + status: ?OverlayStatus, + blocks: BitSeq, + expiry: SecondsSince1970, + manifestCid: ?Cid, +) = + ## Apply merge logic: OR-combine blocks, fallback status, conditional fields. + ## + + overlay.blocks.combineSafe(blocks) + overlay.status = status |? overlay.status + + if manifestCid.isSome: + overlay.manifestCid = manifestCid + + if expiry != ZeroSeconds: + overlay.expiry = expiry + elif overlay.expiry == ZeroSeconds: + overlay.expiry = self.clock.now() + self.overlayTtl + +proc putOverlay*( + self: RepoStore, + treeCid: Cid, + status: ?OverlayStatus = OverlayStatus.none, + blocks = BitSeq.init(0), + expiry = ZeroSeconds, + manifestCid: ?Cid = Cid.none, +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Store overlay metadata with CAS semantics (upsert). + ## On conflict, re-reads current state and merges with provided values + ## instead of blindly overwriting. + ## + ## Parameters: + ## - status: OverlayStatus to set (uses |? fallback if not provided) + ## - blocks: BitSeq to OR-combine with existing + ## - expiry: Expiry time (sets if non-zero, otherwise keeps existing or sets default) + ## - manifestCid: Optional manifest CID to set + ## + + let key = ?overlayKey(treeCid) + self.overlayCache.del(key) + + # Read full KVRecord (preserving CAS token) for correct first attempt + without var record =? (await self.metaDs.get(key, OverlayMetadata)), err: + if not (err of KVStoreKeyNotFound): + trace "Error fetching overlay metadata for put", + treeCid = treeCid, error = err.msg + return failure(err) + record = KVRecord[OverlayMetadata].init(key, OverlayMetadata()) + + self.mergeOverlay(record.val, status, blocks, expiry, manifestCid) + var cachedOverlay = record.val + ?await self.metaDs.tryPut( + record, + maxRetries = 10, + proc( + failed: seq[KVRecord[OverlayMetadata]] + ): Future[?!seq[KVRecord[OverlayMetadata]]] {.async: (raises: [CancelledError]).} = + var records = ?await self.metaDs.get(failed.mapIt(it.key), OverlayMetadata) + self.mergeOverlay(records[0].val, status, blocks, expiry, manifestCid) + cachedOverlay = records[0].val + success records + , + ) + + # cache the final merged overlay + self.overlayCache[key] = cachedOverlay + trace "Overlay metadata stored", treeCid = treeCid, status = cachedOverlay.status + success() + +proc getOverlay*( + self: RepoStore, treeCid: Cid +): Future[?!OverlayMetadata] {.async: (raises: [CancelledError]).} = + ## Get overlay metadata for a dataset. + ## + + let key = ?overlayKey(treeCid) + self.overlayCache.withValue(key, value): + return success value[] + + let meta = ?await self.metaDs.get(key, OverlayMetadata) + trace "OverlayMetadata loaded", treeCid = treeCid, status = meta.val.status + + success meta.val + +proc deleteOverlay*( + self: RepoStore, treeCid: Cid +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Delete overlay metadata + ## + + let key = ?overlayKey(treeCid) + self.overlayCache.del(key) + + ?await self.metaDs.delete(?await self.metaDs.get(key)) + + trace "Overlay metadata deleted", treeCid = treeCid + success() + +func toCid(key: Key): ?!Cid = + success ?Cid.init(key.value).mapFailure + +proc listOverlays*( + self: RepoStore +): Future[?!SafeAsyncIter[Cid]] {.async: (raises: [CancelledError]).} = + ## List all overlay CIDs. + ## + + let + queryKey = ?overlayQueryKey() + iter = ?(await query(self.metaDs, Query.init(queryKey), OverlayMetadata)) + + proc mapCids( + iterRes: ?!(?KVRecord[OverlayMetadata]) + ): Future[?(?!Cid)] {.async: (raises: [CancelledError]).} = + if maybeRecord =? iterRes and record =? maybeRecord: + without cid =? record.key.toCid, err: + trace "Unable to construct Cid from key", error = err.msg + return none(?!Cid) + return some(success(cid)) + return none(?!Cid) + + let safeQueryIter = iter.toSafeAsyncIter() + success await mapFilter[?KVRecord[OverlayMetadata], Cid](safeQueryIter, mapCids) + +proc listOverlaysInState*( + self: RepoStore, status: OverlayStatus +): Future[?!SafeAsyncIter[Cid]] {.async: (raises: [CancelledError]).} = + ## List all overlay CIDs by status. + ## + + let + queryKey = ?overlayQueryKey() + iter = ?(await query(self.metaDs, Query.init(queryKey), OverlayMetadata)) + + let safeQueryIter = iter.toSafeAsyncIter() + + proc filterBytStatus( + iterRes: ?!(?KVRecord[OverlayMetadata]) + ): Future[?(?!Cid)] {.async: (raises: [CancelledError]).} = + if maybeRecord =? iterRes and record =? maybeRecord: + if record.val.status == status: + without cid =? record.key.toCid, err: + trace "Unable to construct Cid from key", error = err.msg + return none(?!Cid) + + return some(success(cid)) + + return none(?!Cid) + + success await mapFilter[?KVRecord[OverlayMetadata], Cid]( + safeQueryIter, filterBytStatus, finishOnErr = false + ) + +proc listOverlaysByExpiry*( + self: RepoStore, limit: int, offset: int +): Future[?!seq[(Cid, OverlayMetadata)]] {.async: (raises: [CancelledError]).} = + ## List overlays sorted by expiry time (ascending). + ## + + let + queryKey = ?overlayQueryKey() + iter = + ?( + await query( + self.metaDs, + Query.init(queryKey, limit = limit, offset = offset), + OverlayMetadata, + ) + ) + + proc mapRecord( + iterRes: ?!(?KVRecord[OverlayMetadata]) + ): Future[?(?!(Cid, OverlayMetadata))] {.async: (raises: [CancelledError]).} = + if maybeRecord =? iterRes and record =? maybeRecord: + without cid =? record.key.toCid, err: + trace "Unable to construct Cid from key", error = err.msg + return none(?!(Cid, OverlayMetadata)) + + return (success((cid, record.val))).some + + let metadata = ( + ?await collect( + await mapFilter[?KVRecord[OverlayMetadata], (Cid, OverlayMetadata)]( + iter.toSafeAsyncIter(), mapRecord + ) + ) + ) + # Sort by expiry (ascending - earliest expiry first) + .sorted( + func (a, b: (Cid, OverlayMetadata)): int = + cmp(a[1].expiry, b[1].expiry) + ) + + success metadata + +proc createTmpOverlay*( + self: RepoStore, expiry = ZeroSeconds +): Future[?!Cid] {.async: (raises: [CancelledError]).} = + var randomBytes: array[32, byte] + Rng.instance[].generate(randomBytes) + let + mhash = ?MultiHash.digest($Sha256HashCodec, randomBytes).mapFailure + tmpTreeCid = ?Cid.init(CIDv1, BlockCodec, mhash).mapFailure + + let expiryTime = + if expiry == ZeroSeconds: + self.clock.now() + self.overlayTtl + else: + expiry + + ?await self.putOverlay( + tmpTreeCid, + status = OverlayStatus.Storing.some, + blocks = BitSeq.init(0), + expiry = expiryTime, + ) + + success tmpTreeCid + +proc dropOverlay*( + self: RepoStore, treeCid: Cid +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Drop an overlay and delete all attached blocks. + ## + ## Queries all leaf records under /meta/leafs/{treeCid}/*, calls + ## delLeafBlockMetadata to decrement refcounts and delete blocks at + ## refCount=0, then deletes the overlay metadata record. + ## + ## Uses a runtime lock to prevent concurrent deletions of the same + ## overlay. Returns success if deletion is already in progress. + ## + + logScope: + treeCid = treeCid + + if treeCid in self.deletingLock: + trace "Overlay deletion already in progress, skipping" + return success() + + self.deletingLock.incl(treeCid) + defer: + self.deletingLock.excl(treeCid) + + trace "Dropping overlay and cleaning up blocks" + + if err =? (await self.putOverlay(treeCid, status = Deleting.some)).errorOption: + error "Unable to mark overlay as deleting", exc = err.msg + return failure(err) + + # Read overlay metadata to get manifestCid before deletion + var manifestCid: ?Cid + if meta =? (await self.getOverlay(treeCid)): + manifestCid = meta.manifestCid + + # Query all leaf records for this tree + let + queryKey = ?blockLeafQueryKey(treeCid) + iter = ?(await query(self.metaDs, Query.init(queryKey), LeafMetadata)) + + # Extract indices from the leaf keys + var indices: seq[Natural] + for recordFut in iter: + if record =? ?catchAsync(?(await recordFut)): + # Key format: /meta/leafs/{treeCid}/{index} + # Extract index from last namespace segment + let indexStr = record.key.value + without idx =? parseInt(indexStr).catch, err: + return failure( + newException( + ValueError, "Invalid index in key: " & indexStr & " - " & err.msg + ) + ) + indices.add(idx.Natural) + + # Delete leaf metadata and decrement refcounts (two-phase atomic) + if indices.len > 0: + ?await delLeafBlockMetadata(self, treeCid, indices) + trace "Deleted leaf metadata and updated refcounts", count = indices.len + + # Delete overlay metadata + ?await self.deleteOverlay(treeCid) + + if cid =? manifestCid: + let + key = ?makePrefixKey(self.postFixLen, cid) + record = ?await self.repoDs.get(key) + + ?await self.repoDs.tryDelete(record) + + trace "Overlay dropped successfully" + + success() + +proc finalizeOverlay*( + self: RepoStore, + tmpCid, realTreeCid: Cid, + status = OverlayStatus.none, + expiry = ZeroSeconds, +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Promote a temp overlay to a real overlay. + ## + ## Atomically moves all leaf records and overlay metadata from + ## tmpCid to realTreeCid in a single transaction, then updates + ## the overlay metadata (expiry/status) as a separate CAS operation. + ## + ## Block metadata is unchanged (keyed by blkCid, not treeCid). + ## + + logScope: + tmpCid = tmpCid + realTreeCid = realTreeCid + + trace "Finalizing temp overlay" + + let + tmpOverlayKey = ?overlayKey(tmpCid) + overlayKey = ?overlayKey(realTreeCid) + + defer: + self.overlayCache.del(tmpOverlayKey) + + # Atomically move both leaf records and overlay metadata + if err =? ( + await self.metaDs.moveKeysAtomic( + @[ + (?(BlockLeafKey / $tmpCid), ?(BlockLeafKey / $realTreeCid)), + (tmpOverlayKey, overlayKey), + ] + ) + ).errorOption: + if err of KVConflictError: + # Destination already has the data (content-addressed guarantee). + # Nothing was moved - tmp is still intact. Drop it cleanly. + trace "Overlay already exists at realTreeCid, dropping tmp" + if dropErr =? (await noCancel self.dropOverlay(tmpCid)).errorOption: + error "Unable to drop tmp overlay after finalize conflict", exc = dropErr.msg + return success() + error "Unable to move overlay metadata atomically", exc = err.msg + return failure(err) + + # Update overlay metadata (expiry/status) at the new location. + # This is a separate CAS operation - if it fails, data is safe + # (everything is already at realTreeCid) and retry will find it. + let expiryTime = + if expiry == ZeroSeconds: + self.clock.now() + self.overlayTtl + else: + expiry + + if err =? + (await self.putOverlay(realTreeCid, status = status, expiry = expiryTime)).errorOption: + error "Unable to update overlay metadata after finalization", exc = err.msg + return failure(err) + + trace "Temp overlay finalized successfully" + success() + +proc withOverlay*[T]( + self: RepoStore, + treeCid: Cid, + status = OverlayStatus.none, + expiry = ZeroSeconds, + body: proc(): Future[?!T] {.closure, gcsafe, async: (raises: [CancelledError]).}, +): Future[?!T] {.async: (raises: [CancelledError]).} = + ## Create or update overlay with initial state and expiry. + ## + + logScope: + treeCid = treeCid + status = status + + trace "Starting overlay operation" + if err =? (await self.putOverlay(treeCid, status, BitSeq.init(0), expiry)).errorOption: + error "Unable to create/update overlay metadata", exc = err.msg + return failure(err) + + let + bodyRes = await body() + finalState = if bodyRes.isOk: Completed.some else: Failure.some + + if err =? + (await self.putOverlay(treeCid, finalState, BitSeq.init(0), expiry)).errorOption: + error "Unable to set overlay final state", exc = err.msg + return failure(err) + + trace "Overlay operation completed", finalState + + return bodyRes + +proc withTmpOverlay*( + self: RepoStore, + body: proc(tmpCid: Cid): Future[?!Cid] {. + closure, gcsafe, async: (raises: [CancelledError]) + .}, +): Future[?!Cid] {.async: (raises: [CancelledError]).} = + ## Create a temporary overlay, run body, finalize or drop. + ## Body must return ?!Cid (the real treeCid). + ## + + trace "Starting temporary overlay operation" + + var completed = false + + without tmpCid =? (await self.createTmpOverlay()), err: + error "Unable to create temporary overlay", exc = err.msg + return failure(err) + + trace "Temporary overlay created", tmpCid + + defer: + if not completed: + # Tmp overlays have no refcount associations yet - safe to drop directly + if dropErr =? (await noCancel self.dropOverlay(tmpCid)).errorOption: + error "Unable to drop tmp overlay on error", exc = dropErr.msg + + let bodyRes = await body(tmpCid) + + without realCid =? bodyRes, err: + error "Body failed to return real tree CID", error = err.msg + return failure(err) + + completed = true + trace "Body completed successfully, finalizing overlay", realCid, tmpCid + if finalErr =? ( + await self.finalizeOverlay(tmpCid, realCid, status = OverlayStatus.Completed.some) + ).errorOption: + error "Unable to finalize tmp overlay", exc = finalErr.msg + return failure(finalErr) + + bodyRes diff --git a/archivist/stores/repostore/store.nim b/archivist/stores/repostore/store.nim index 27fc23bd..cf4d5c4d 100644 --- a/archivist/stores/repostore/store.nim +++ b/archivist/stores/repostore/store.nim @@ -9,27 +9,28 @@ {.push raises: [].} +import std/sets +import std/tables + import pkg/chronos -import pkg/chronos/futures -import pkg/datastore -import pkg/datastore/typedds -import pkg/libp2p/[cid, multicodec] +import pkg/kvstore +import pkg/libp2p/cid import pkg/questionable import pkg/questionable/results +import pkg/stew/bitseqs -import ./coders +import ./overlays import ./types import ./operations import ../blockstore import ../keyutils -import ../queryiterhelper import ../../blocktype -import ../../clock import ../../logutils import ../../merkletree import ../../utils +import ../../manifest -export blocktype, cid +export blocktype, cid, overlays logScope: topics = "archivist repostore" @@ -38,185 +39,266 @@ logScope: # BlockStore API ########################################################### -method getBlock*( - self: RepoStore, cid: Cid -): Future[?!Block] {.async: (raises: [CancelledError]).} = - ## Get a block from the blockstore +proc getBlocksBitmap*( + self: RepoStore, treeCid: Cid +): Future[?!BitSeq] {.async: (raises: [CancelledError]).} = + ## Get the overlay BitSeq for a given tree CID. + ## + ## Returns the bitmap of stored block indices. Use for batch + ## presence checks (e.g., in fetchBatched) to avoid per-block + ## overlay metadata fetches. Returns empty BitSeq if overlay + ## doesn't exist. ## - logScope: - cid = cid + let key = ?overlayKey(treeCid) - if cid.isEmpty: - trace "Empty block, ignoring" - return cid.emptyBlock + self.overlayCache.withValue(key, cached): + trace "Overlay cache hit", treeCid + return success(cached[].blocks) - without key =? makePrefixKey(self.postFixLen, cid), err: - trace "Error getting key from provider", err = err.msg + without overlayMeta =? await self.metaDs.get(key, OverlayMetadata), err: + if err of KVStoreKeyNotFound: + trace "Overlay not found, returning empty", treeCid + return success(BitSeq.init(0)) return failure(err) - without data =? await self.repoDs.get(key), err: - if not (err of DatastoreKeyNotFound): - trace "Error getting block from datastore", err = err.msg, key - return failure(err) + # Re-check cache: a concurrent write may have populated it while we awaited + self.overlayCache.withValue(key, fresher): + trace "Overlay cache populated during await, using cached", treeCid + return success(fresher[].blocks) + self.overlayCache[key] = overlayMeta.val + trace "Overlay found", treeCid, bitmapLen = overlayMeta.val.blocks.len + success(overlayMeta.val.blocks) - return failure(newException(BlockNotFoundError, err.msg)) +proc checkBitmap*( + self: RepoStore, treeCid: Cid, index: Natural +): Future[?!bool] {.async: (raises: [CancelledError]).} = + ## Check if the overlay BitSeq has the bit set for the given index. + ## + ## Returns true if the bit is set or if the overlay doesn't exist + ## (caller should proceed with normal lookup). Returns false if the + ## bit is not set (block is definitely absent - caller can fast-reject). + ## - trace "Got block for cid", cid - return Block.new(cid, data, verify = true) + let bits = ?await self.getBlocksBitmap(treeCid) + if index >= bits.len or not bits[index]: + trace "Block not in overlay BitSeq, fast-path rejection", treeCid, index + return success(false) -method getBlockAndProof*( - self: RepoStore, treeCid: Cid, index: Natural -): Future[?!(Block, ArchivistProof)] {.async: (raises: [CancelledError]).} = - without leafMd =? await self.getLeafMetadata(treeCid, index), err: - return failure(err) + success(true) - without blk =? await self.getBlock(leafMd.blkCid), err: - return failure(err) +method getBlocks*( + self: RepoStore, cids: seq[Cid] +): Future[?!seq[Block]] {.async: (raises: [CancelledError]).} = + ## Get multiple blocks by CID from the blockstore (no tree context). + ## Empty CIDs are synthesized directly. Missing blocks are silently omitted. + ## - success((blk, leafMd.proof)) + if cids.len == 0: + return success(newSeq[Block]()) -method getBlock*( - self: RepoStore, treeCid: Cid, index: Natural -): Future[?!Block] {.async: (raises: [CancelledError]).} = - without leafMd =? await self.getLeafMetadata(treeCid, index), err: - return failure(err) + var + keyToCid: Table[Key, Cid] + blocks: seq[Block] - await self.getBlock(leafMd.blkCid) + for cid in cids: + if cid.isEmpty: + blocks.add(?cid.emptyBlock) + continue -method getBlock*( - self: RepoStore, address: BlockAddress -): Future[?!Block] {.async: (raw: true, raises: [CancelledError]).} = - ## Get a block from the blockstore + let key = ?makePrefixKey(self.postFixLen, cid) + keyToCid[key] = cid + + if keyToCid.len > 0: + let records = ?await self.repoDs.get(toSeq(keyToCid.keys)) + for record in records: + let cid = ?catch(keyToCid[record.key]) + blocks.add(?Block.new(cid, record.val, verify = true)) + + success(blocks) + +method getBlocks*( + self: RepoStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Natural, Block)]] {.async: (raises: [CancelledError]).} = + ## Get multiple blocks by tree CID and indices as a batch. + ## Delegates to getBlocksAndProofs and strips proofs. ## - if address.leaf: - self.getBlock(address.treeCid, address.index) - else: - self.getBlock(address.cid) + success (?await self.getBlocksAndProofs(treeCid, indices)).mapIt((it[0], it[1])) -method ensureExpiry*( - self: RepoStore, cid: Cid, expiry: SecondsSince1970 +method putCidsAndProofs*( + self: RepoStore, treeCid: Cid, items: seq[(Natural, Cid, ArchivistProof)] ): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Ensure that block's associated expiry is at least given timestamp - ## If the current expiry is lower then it is updated to the given one, otherwise it is left intact + ## Put multiple CIDs and proofs as a batch ## - if expiry <= 0: - return - failure(newException(ValueError, "Expiry timestamp must be larger then zero")) + logScope: + treeCid = treeCid + totalItems = items.len - await self.updateBlockMetadata(cid, minExpiry = expiry) + trace "Storing batch Leaf and Block Metadata" -method ensureExpiry*( - self: RepoStore, treeCid: Cid, index: Natural, expiry: SecondsSince1970 -): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Ensure that block's associated expiry is at least given timestamp - ## If the current expiry is lower then it is updated to the given one, otherwise it is left intact + if items.len == 0: + return success() + + if err =? (await self.putLeafBlockMeta(treeCid, items)).errorOption: + trace "Unable to store batch Leaf and Block Metadata", err = err.msg + return failure(err) + + return success() + +method putCellCidsAndProofs*( + self: RepoStore, treeCid: Cid, items: seq[(Natural, Cid, Cid, ArchivistProof)] +): Future[?!void] {.async: (raises: [CancelledError]), gcsafe.} = + ## Put multiple cell CIDs and proofs as a batch for slot proof trees. + ## Each item is (index, cellCid, blkCid, proof). ## - without leafMd =? await self.getLeafMetadata(treeCid, index), err: + logScope: + treeCid = treeCid + totalItems = items.len + + trace "Storing batch Leaf and Block Metadata" + + if items.len == 0: + return success() + + if err =? (await self.putCellLeafBlockMeta(treeCid, items)).errorOption: + trace "Unable to store batch Leaf and Block Metadata", err = err.msg return failure(err) - await self.ensureExpiry(leafMd.blkCid, expiry) + return success() -method putCidAndProof*( - self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: ArchivistProof +method putBlocks*( + self: RepoStore, treeCid: Cid, items: seq[(Block, Natural, ArchivistProof)] ): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Put a block to the blockstore + ## Put multiple leafs and blocks as a batch (primary method) ## logScope: treeCid = treeCid - index = index - blkCid = blkCid + totalItems = items.len - trace "Storing LeafMetadata" + trace "Storing Leafs and Blocks" - without res =? await self.putLeafMetadata(treeCid, index, blkCid, proof), err: - return failure(err) + if items.len == 0: + return success() - if blkCid.mcodec == BlockCodec: - if res == Stored: - if err =? (await self.updateBlockMetadata(blkCid, plusRefCount = 1)).errorOption: - return failure(err) - trace "Leaf metadata stored, block refCount incremented" - else: - trace "Leaf metadata already exists" + var + totalSize = 0 + uniqueBlks: HashSet[Block] # filter out duplicate leafs for different tree branches - return success() + let blocks = collect(newSeq): + for (blk, idx, proof) in items.deduplicate(): + if not blk.cid.isEmpty: + totalSize += blk.data.len + uniqueBlks.incl(blk) + (idx, blk.cid, proof) -method getCidAndProof*( - self: RepoStore, treeCid: Cid, index: Natural -): Future[?!(Cid, ArchivistProof)] {.async: (raises: [CancelledError]).} = - without leafMd =? await self.getLeafMetadata(treeCid, index), err: + trace "Putting blocks", actualBlocks = blocks.len, totalSize + + if not self.available(totalSize.NBytes): + return failure(newException(QuotaNotEnoughError, "Blocks would exceed quota!")) + + # Atomic metadata update (leaf + block refcount) + if err =? (await self.putLeafBlockMeta(treeCid, blocks)).errorOption: + trace "Unable to store Leaf and Block Metadata", err = err.msg return failure(err) - success((leafMd.blkCid, leafMd.proof)) + trace "Writting blocks to disc", actualBlocks = blocks.len + # Write blocks to FS (best effort, idempotent) + # Build records and capture sizes before moving into put + var + records = uniqueBlks.mapIt( + RawKVRecord.init(?makePrefixKey(self.postFixLen, it.cid), it.data) + ) + keySizes = records.mapIt((it.key, it.val.len)) + + let skipped = (?await self.repoDs.put(move(records))).toHashSet + + # Count only unique blocks that were successfully written + var newBlocks, newBytes = 0 + for (key, size) in keySizes: + if key notin skipped: + newBytes += size + newBlocks += 1 + + if newBlocks > 0: + ?await self.updateCounters(quotaDelta = newBytes, blocksDelta = newBlocks) + + if onBlock =? self.onBlockStored: + await allFutures(uniqueBlks.mapIt(onBlock(it.cid))) + + return success() method getCid*( self: RepoStore, treeCid: Cid, index: Natural ): Future[?!Cid] {.async: (raises: [CancelledError]).} = - without leafMd =? await self.getLeafMetadata(treeCid, index), err: - return failure(err) - - success(leafMd.blkCid) + success (?await self.getLeafMetadata(treeCid, index)).blkCid -method putBlock*( - self: RepoStore, blk: Block, ttl = Duration.none +proc putBlockInternal( + self: RepoStore, blk: Block ): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Put a block to the blockstore - ## - logScope: cid = blk.cid - let expiry = self.clock.now() + (ttl |? self.blockTtl).seconds - - without res =? await self.storeBlock(blk, expiry), err: - return failure(err) - - if res.kind == Stored: - trace "Block Stored" - if err =? (await self.updateQuotaUsage(plusUsed = res.used)).errorOption: - # rollback changes - without delRes =? await self.tryDeleteBlock(blk.cid), err: - return failure(err) + if blk.cid.isEmpty: + trace "Skipping empty block" + return success() + + if not self.available(blk.data.len.NBytes): + return failure(newException(QuotaNotEnoughError, "Block would exceed quota!")) + + if err =? ( + await self.metaDs.put( + ?blockMetaKey(blk.cid), BlockMetadata(refCount: 0, cid: blk.cid) + ) + ).errorOption: + if err of KVConflictError: + trace "Block metadata already exists", err = err.msg + # don't return here, allow writing the block on disk + else: + trace "Error writing block metadata", err = err.msg return failure(err) - if err =? (await self.updateTotalBlocksCount(plusCount = 1)).errorOption: + let blkKey = ?makePrefixKey(self.postFixLen, blk.cid) + + # we never update blocks on fs, we only insert + if err =? (await self.repoDs.put(RawKVRecord.init(blkKey, blk.data))).errorOption: + if err of KVConflictError: + trace "Block already in store", err = err.msg + return success() + else: + trace "Error writing block on disk", err = err.msg return failure(err) - if onBlock =? self.onBlockStored: - await onBlock(blk.cid) - else: - trace "Block already exists" + ?await self.updateCounters(quotaDelta = blk.data.len, blocksDelta = 1) + if onBlock =? self.onBlockStored: + await onBlock(blk.cid) return success() -proc delBlockInternal( - self: RepoStore, cid: Cid -): Future[?!DeleteResultKind] {.async: (raises: [CancelledError]).} = - logScope: - cid = cid - - if cid.isEmpty: - return success(Deleted) +method putBlock*( + self: RepoStore, blk: Block, ttl = Duration.none +): Future[?!void] {. + async: (raises: [CancelledError], raw: true), + deprecated: + "Use putBlock(treeCid: Cid, items: seq[(Block, Natural, ArchivistProof)])" +.} = + putBlockInternal(self, blk) - trace "Attempting to delete a block" +proc blockRefCount*( + self: RepoStore, cid: Cid +): Future[?!Natural] {.async: (raises: [CancelledError]).} = + ## Returns the reference count for a block. If the count is zero; + ## this means the block is eligible for garbage collection. + ## - without res =? await self.tryDeleteBlock(cid, self.clock.now()), err: + without blockMeta =? (await self.metaDs.get(?blockMetaKey(cid), BlockMetadata)), err: + trace "Unable to retrieve block metadata", err = err.msg return failure(err) - if res.kind == Deleted: - trace "Block deleted" - if err =? (await self.updateTotalBlocksCount(minusCount = 1)).errorOption: - return failure(err) - - if err =? (await self.updateQuotaUsage(minusUsed = res.released)).errorOption: - return failure(err) - - success(res.kind) + success blockMeta.val.refCount method delBlock*( self: RepoStore, cid: Cid @@ -227,41 +309,25 @@ method delBlock*( logScope: cid = cid - without outcome =? await self.delBlockInternal(cid), err: - return failure(err) - - case outcome - of InUse: - failure("Directly deleting a block that is part of a dataset is not allowed.") - of NotFound: - trace "Block not found, ignoring" - success() - of Deleted: - trace "Block already deleted" - success() - -method delBlock*( - self: RepoStore, treeCid: Cid, index: Natural -): Future[?!void] {.async: (raises: [CancelledError]).} = - without leafMd =? await self.getLeafMetadata(treeCid, index), err: - if err of BlockNotFoundError: - return success() - else: - return failure(err) + if cid.isEmpty: + trace "Skipping empty Cid" + return success() - if err =? (await self.delLeafMetadata(treeCid, index)).errorOption: - error "Failed to delete leaf metadata, block will remain on disk.", err = err.msg + without refCount =? (await self.blockRefCount(cid)), err: + if err of KVStoreKeyNotFound: + return failure(newException(BlockNotFoundError, err.msg)) return failure(err) - if err =? - (await self.updateBlockMetadata(leafMd.blkCid, minusRefCount = 1)).errorOption: - if not (err of BlockNotFoundError): - return failure(err) + if refCount > 0.Natural: + return + failure("Directly deleting a block that is part of a dataset is not allowed.") - without _ =? await self.delBlockInternal(leafMd.blkCid), err: - return failure(err) + let skipped = ?await self.tryDeleteBlocks(cid) + if skipped.len > 0: + trace "Some blocks were not deleted, likely due to refCount > 0", + skipped = skipped.len - success() + return success() method hasBlock*( self: RepoStore, cid: Cid @@ -273,7 +339,7 @@ method hasBlock*( cid = cid if cid.isEmpty: - trace "Empty block, ignoring" + trace "Skipping empty block" return success true without key =? makePrefixKey(self.postFixLen, cid), err: @@ -282,16 +348,31 @@ method hasBlock*( return await self.repoDs.has(key) -method hasBlock*( - self: RepoStore, treeCid: Cid, index: Natural -): Future[?!bool] {.async: (raises: [CancelledError]).} = - without leafMd =? await self.getLeafMetadata(treeCid, index), err: - if err of BlockNotFoundError: - return success(false) - else: - return failure(err) +method hasBlocks*( + self: RepoStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Natural, bool)]] {.async: (raises: [CancelledError]).} = + ## Check if multiple blocks exist in the overlay bitmap. + ## + ## We use the bitmap exclusively for fast checks. By + ## definition there is no guarantee that a block we + ## deemed as present may have been deleted between operations. + ## The ultimate source of truth is the blocks store itself - + ## either the block is on disk and it will be returned with + ## a getBlock* variant, or it isn't and it needs to be recovered + ## or downloaded. Results include the index for caller matching; + ## ordering is not guaranteed. + ## + + if indices.len == 0: + return success(newSeq[(Natural, bool)]()) - await self.hasBlock(leafMd.blkCid) + let indices = indices.deduplicate() + + var results: seq[(Natural, bool)] + for idx in indices: + results.add((idx, ?await self.checkBitmap(treeCid, idx.Natural))) + + success(results) method listBlocks*( self: RepoStore, blockType = BlockType.Manifest @@ -313,68 +394,169 @@ method listBlocks*( proc next(): Future[?!Cid] {.async: (raises: [CancelledError]).} = await idleAsync() - if pair =? (await queryIter.next()): - if cid =? pair.key: - doAssert pair.data.len == 0 - trace "Retrieved record from repo", cid - return Cid.init(cid.value).mapFailure + if maybeRecord =? (await queryIter.next()): + if record =? maybeRecord: + trace "Retrieved record from repo", key = record.key + return Cid.init(record.key.value).mapFailure return Cid.failure("No or invalid Cid") proc isFinished(): bool = queryIter.finished - return success SafeAsyncIter[Cid].new(next, isFinished) + proc onDispose(): Future[?!void] {.async: (raises: []).} = + # kvquery.dispose returns Future[?!void] - await and propagate errors + return await dispose(queryIter) -proc createBlockExpirationQuery(maxNumber: int, offset: int): ?!Query = - let queryKey = ?createBlockExpirationMetadataQueryKey() - success Query.init(queryKey, offset = offset, limit = maxNumber) + proc isDisposed(): bool = + queryIter.disposed -proc blockRefCount*(self: RepoStore, cid: Cid): Future[?!Natural] {.async.} = - ## Returns the reference count for a block. If the count is zero; - ## this means the block is eligible for garbage collection. + return success SafeAsyncIter[Cid].new(next, isFinished, onDispose, isDisposed) + +method delBlocks*( + self: RepoStore, treeCid: Cid, indices: seq[Natural] +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Delete multiple blocks by tree CID and indices as a batch. + ## + ## Uses delLeafBlockMetadata which performs atomic delete of leaf + ## metadata and block refcount updates. ## - without key =? createBlockExpirationMetadataKey(cid), err: - return failure(err) - without md =? await get[BlockMetadata](self.metaDs, key), err: - return failure(err) + if indices.len == 0: + return success() - return success(md.refCount) + logScope: + treeCid = treeCid + count = indices.len -method getBlockExpirations*( - self: RepoStore, maxNumber: int, offset: int -): Future[?!SafeAsyncIter[BlockExpiration]] {. - async: (raises: [CancelledError]), base, gcsafe -.} = - ## Get iterator with block expirations + trace "Batch deleting blocks" + + ?await self.delLeafBlockMetadata(treeCid, indices) + success() + +method getBlocksAndProofs*( + self: RepoStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Natural, Block, ArchivistProof)]] {.async: (raises: [CancelledError]).} = + ## Get multiple blocks and proofs as a batch. + ## + ## Fetches overlay bitmap once, filters indices, then batch-fetches + ## leaf metadata and block data in two round-trips total. + ## Missing blocks are silently omitted from the result. ## - without beQuery =? createBlockExpirationQuery(maxNumber, offset), err: - error "Unable to format block expirations query", err = err.msg - return failure(err) + if indices.len == 0: + return success(newSeq[(Natural, Block, ArchivistProof)]()) - without queryIter =? await query[BlockMetadata](self.metaDs, beQuery), err: - error "Unable to execute block expirations query", err = err.msg - return failure(err) + logScope: + treeCid = treeCid + count = indices.len + + trace "Batch getting blocks and proofs" + + # Filter to indices present in bitmap + var presentIndices: seq[Natural] + for idx in indices: + if ?await self.checkBitmap(treeCid, idx.Natural): + presentIndices.add(idx) + + if presentIndices.len == 0: + return success(newSeq[(Natural, Block, ArchivistProof)]()) + + # Batch-fetch leaf metadata for all present indices + var leafKeys: seq[Key] + for idx in presentIndices: + leafKeys.add(?blockLeafKey(treeCid, idx)) + + let leafRecords = ?await self.metaDs.get(leafKeys, LeafMetadata) + + trace "Leaf metadata fetched", + treeCid, leafKeysLen = leafKeys.len, leafRecordsLen = leafRecords.len + + # Collect block data keys from leaf records. + # leafRecords may be a subset of leafKeys (missing silently skipped). + # Empty-CID leaves (erasure padding) are synthesized directly without + # repoDs lookup - they are never stored but callers expect them back. + var + keyToCidProof: Table[Key, (Natural, Cid, ArchivistProof)] + results: seq[(Natural, Block, ArchivistProof)] + for record in leafRecords: + let blkCid = record.val.blkCid + + if blkCid.isEmpty: + let index = ?catch(parseInt(record.key.value)) + results.add((index.Natural, ?blkCid.emptyBlock, record.val.proof)) + continue + + let + index = ?catch(parseInt(record.key.value)) + proof = record.val.proof + key = ?makePrefixKey(self.postFixLen, blkCid) + keyToCidProof[key] = (index.Natural, blkCid, proof) + + if keyToCidProof.len > 0: + # Batch-fetch block data from FS. + let blkRecords = ?await self.repoDs.get(toSeq(keyToCidProof.keys)) + + for record in blkRecords: + let (idx, cid, proof) = ?catch(keyToCidProof[record.key]) + results.add((idx, ?Block.new(cid, record.val, verify = false), proof)) + + trace "Batch got blocks and proofs", found = results.len + success(results) + +method getCidsAndProofs*( + self: RepoStore, treeCid: Cid, indices: seq[Natural] +): Future[?!seq[(Cid, ArchivistProof)]] {.async: (raises: [CancelledError]).} = + ## Get multiple CIDs and proofs as a batch. + ## + ## Fetches overlay bitmap once, filters indices, then batch-fetches + ## leaf metadata. No block data fetch needed - only metadata. + ## Missing entries are silently omitted from the result. + ## + + if indices.len == 0: + return success(newSeq[(Cid, ArchivistProof)]()) + + logScope: + treeCid = treeCid + count = indices.len + + trace "Batch getting CIDs and proofs" - let filteredIter: SafeAsyncIter[KeyVal[BlockMetadata]] = - await queryIter.toSafeAsyncIter().filterSuccess() + # Filter to indices present in bitmap + var presentIndices: seq[Natural] + for idx in indices: + if ?await self.checkBitmap(treeCid, idx.Natural): + presentIndices.add(idx) - proc mapping( - kvRes: ?!KeyVal[BlockMetadata] - ): Future[Option[?!BlockExpiration]] {.async: (raises: [CancelledError]).} = - without kv =? kvRes, err: - error "Error occurred when getting KeyVal", err = err.msg - return Result[BlockExpiration, ref CatchableError].none - without cid =? Cid.init(kv.key.value).mapFailure, err: - error "Failed decoding cid", err = err.msg - return Result[BlockExpiration, ref CatchableError].none + if presentIndices.len == 0: + return success(newSeq[(Cid, ArchivistProof)]()) - some(success(BlockExpiration(cid: cid, expiry: kv.value.expiry))) + # Batch-fetch leaf metadata for all present indices + var leafKeys: seq[Key] + for idx in presentIndices: + leafKeys.add(?blockLeafKey(treeCid, idx)) - let blockExpIter = - await mapFilter[KeyVal[BlockMetadata], BlockExpiration](filteredIter, mapping) - success(blockExpIter) + let leafRecords = ?await self.metaDs.get(leafKeys, LeafMetadata) + + trace "Leaf metadata fetched", + treeCid, leafKeysLen = leafKeys.len, leafRecordsLen = leafRecords.len + + # Extract CID and proof from each leaf record + var results: seq[(Cid, ArchivistProof)] + for record in leafRecords: + results.add((record.val.blkCid, record.val.proof)) + + trace "Batch got CIDs and proofs", found = results.len + success(results) + +method putBlocks*( + self: RepoStore, treeCid: Cid, blocks: seq[(Natural, Block)] +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Put multiple blocks without proofs (for non-leaf blocks) + ## + for (index, blk) in blocks: + ?await self.putBlock(treeCid, blk, index, nil) + success() method close*(self: RepoStore): Future[void] {.async: (raises: []).} = ## Close the blockstore, cleaning up resources managed by it. @@ -384,13 +566,12 @@ method close*(self: RepoStore): Future[void] {.async: (raises: []).} = trace "Closing repostore" if not self.metaDs.isNil: - try: - (await noCancel self.metaDs.close()).expect("Should meta datastore") - except CatchableError as err: - error "Failed to close meta datastore", err = err.msg + if err =? (await noCancel self.metaDs.close()).errorOption: + error "Failed to close metadata store!", err = err.msg if not self.repoDs.isNil: - (await noCancel self.repoDs.close()).expect("Should repo datastore") + if err =? (await noCancel self.repoDs.close()).errorOption: + error "Failed to close repods store!", err = err.msg ########################################################### # RepoStore procs @@ -404,7 +585,10 @@ proc reserve*( trace "Reserving bytes", bytes - await self.updateQuotaUsage(plusReserved = bytes) + if not self.available(bytes): + return failure(newException(QuotaNotEnoughError, "Not enough bytes to reserve!")) + + await self.updateCounters(reservedDelta = bytes.int) proc release*( self: RepoStore, bytes: NBytes @@ -413,8 +597,41 @@ proc release*( ## trace "Releasing bytes", bytes + if bytes > self.quotaReservedBytes: + return failure(newException(QuotaNotEnoughError, "Not enough bytes to release!")) + + await self.updateCounters(reservedDelta = -(bytes.int)) + +method storeManifest*( + self: RepoStore, manifest: Manifest +): Future[?!Block] {.async: (raises: [CancelledError]), gcsafe.} = + let manifestBlk = ?manifest.toBlock + + if err =? ( + await self.putOverlay( + treeCid = manifest.treeCid, manifestCid = manifestBlk.cid.some + ) + ).errorOption: + trace "Unable to set manifestCid for overlay metadata", + treeCid = manifest.treeCid, manifestCid = manifestBlk.cid + return failure(err) - await self.updateQuotaUsage(minusReserved = bytes) + ?await self.putBlockInternal(manifestBlk) + trace "Stored manifest block", cid = manifestBlk.cid + + success manifestBlk + +method fetchManifest*( + self: RepoStore, cid: Cid +): Future[?!Manifest] {.async: (raises: [CancelledError]), gcsafe.} = + ## Fetch and decode a manifest from the blockstore. + ## Retrieves the block by CID and decodes it as a Manifest. + ## + + without blk =? await self.getBlock(cid), err: + return failure(err) + + Manifest.decode(blk.data) proc start*( self: RepoStore @@ -426,13 +643,7 @@ proc start*( trace "Repo already started" return - trace "Starting rep" - if err =? (await self.updateTotalBlocksCount()).errorOption: - raise newException(ArchivistError, err.msg) - - if err =? (await self.updateQuotaUsage()).errorOption: - raise newException(ArchivistError, err.msg) - + trace "Starting repo" self.started = true proc stop*(self: RepoStore): Future[void] {.async: (raises: []).} = diff --git a/archivist/stores/repostore/types.nim b/archivist/stores/repostore/types.nim index ec99b842..e561ed37 100644 --- a/archivist/stores/repostore/types.nim +++ b/archivist/stores/repostore/types.nim @@ -7,11 +7,14 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +import std/sets +import std/tables + import pkg/chronos -import pkg/datastore -import pkg/datastore/typedds +import pkg/kvstore import pkg/libp2p/cid import pkg/questionable +import pkg/stew/bitseqs import ../blockstore import ../../clock @@ -21,56 +24,93 @@ import ../../systemclock import ../../units const - DefaultBlockTtl* = 30.days + DefaultOverlayTtl* = SecondsSince1970 30.days.seconds # ttl in seconds DefaultQuotaBytes* = 20.GiBs + ZeroSeconds* = SecondsSince1970 0 type QuotaNotEnoughError* = object of ArchivistError + OverlayDeletingError* = object of ArchivistError RepoStore* = ref object of BlockStore postFixLen*: int - repoDs*: Datastore - metaDs*: TypedDatastore + repoDs*: KVStore + metaDs*: KVStore clock*: Clock quotaMaxBytes*: NBytes quotaUsage*: QuotaUsage totalBlocks*: Natural - blockTtl*: Duration + overlayTtl*: SecondsSince1970 started*: bool + deletingLock*: HashSet[Cid] + overlayCache*: Table[Key, OverlayMetadata] QuotaUsage* {.serialize.} = object used*: NBytes reserved*: NBytes BlockMetadata* {.serialize.} = object - expiry*: SecondsSince1970 - size*: NBytes + cid*: Cid refCount*: Natural LeafMetadata* {.serialize.} = object + deleted*: bool blkCid*: Cid proof*: ArchivistProof - - BlockExpiration* {.serialize.} = object - cid*: Cid - expiry*: SecondsSince1970 - - DeleteResultKind* {.serialize.} = enum - Deleted = 0 # block removed from store - InUse = 1 # block not removed, refCount > 0 and not expired - NotFound = 2 # block not found in store - - DeleteResult* {.serialize.} = object - kind*: DeleteResultKind - released*: NBytes - - StoreResultKind* {.serialize.} = enum - Stored = 0 # new block stored - AlreadyInStore = 1 # block already in store - - StoreResult* {.serialize.} = object - kind*: StoreResultKind - used*: NBytes + case isCell*: bool + of true: + cellCid*: Cid + else: + discard + + OverlayStatus* {.serialize.} = enum + Pending ## Initial state, not yet active + Failure ## Unrecoverable error + Storing ## Upload/Download in progress + Downloading ## Download in progress (active) + Repairing ## Repair in progress + Completed ## All blocks received/stored + Deleting ## Deletion in progress + + CleanupMode* {.serialize.} = enum + ## Mode for cleaning up after storage request + SlotsOnly ## Delete slot overlays, keep dataset + Full ## Delete both slots and dataset + None ## Keep everything + + OverlayMetadata* {.serialize.} = object + ## Transient local state for an overlay + ## + ## - protected=false -> original dataset + ## - protected=true, verifiable=false -> protected dataset + ## - protected=true, verifiable=true -> slot + ## + ## BitSeq semantics (blocks field): + ## + ## The bitmap is a bloom-filter-like optimization to avoid unnecessary + ## metadata/FS lookups: + ## + ## - bit NOT set -> block is DEFINITELY absent (fast-path rejection) + ## - bit SET -> block is PROBABLY present (must verify via FS) + ## + ## The FS blob store is the ultimate source of truth - a block is + ## present if and only if it physically exists on disk. The bitmap + ## is set atomically with metadata before the FS write, so a crash + ## between metadata commit and FS write can leave a bit set for a + ## block that was never persisted. This is acceptable: the read path + ## falls through to FS, discovers the block is missing, and should + ## treat it as absent (and may clear the stale bit). + ## + ## Invariants: + ## - Length = max_index_stored + 1 (dynamically grows via combineSafe) + ## - Bits are set in putLeafBlockMetaImpl (atomic with metadata) + ## - Bits are cleared in delLeafBlockMetadata (atomic with metadata) + ## - On FS miss for a set bit, callers treat as absent + ## + status*: OverlayStatus + expiry*: SecondsSince1970 # overlay expiration + blocks*: BitSeq # bitmap of currently stored blocks + manifestCid*: ?Cid # CID of the manifest block (for cleanup) func quotaUsedBytes*(self: RepoStore): NBytes = self.quotaUsage.used @@ -85,25 +125,25 @@ func available*(self: RepoStore): NBytes = return self.quotaMaxBytes - self.totalUsed func available*(self: RepoStore, bytes: NBytes): bool = - return bytes < self.available() + return bytes <= self.available() func new*( T: type RepoStore, - repoDs: Datastore, - metaDs: Datastore, + repoDs: KVStore, + metaDs: KVStore, clock: Clock = SystemClock.new(), postFixLen = 2, quotaMaxBytes = DefaultQuotaBytes, - blockTtl = DefaultBlockTtl, + overlayTtl = DefaultOverlayTtl, ): RepoStore = ## Create new instance of a RepoStore ## RepoStore( repoDs: repoDs, - metaDs: TypedDatastore.init(metaDs), + metaDs: metaDs, clock: clock, postFixLen: postFixLen, quotaMaxBytes: quotaMaxBytes, - blockTtl: blockTtl, + overlayTtl: overlayTtl, onBlockStored: CidCallback.none, ) diff --git a/archivist/stores/treehelper.nim b/archivist/stores/treehelper.nim index b2f45f1a..1888ecbb 100644 --- a/archivist/stores/treehelper.nim +++ b/archivist/stores/treehelper.nim @@ -16,6 +16,8 @@ import pkg/metrics import pkg/questionable import pkg/questionable/results +import pkg/libp2p/cid + import ./blockstore import ../utils/asynciter import ../merkletree @@ -23,9 +25,9 @@ import ../merkletree proc putSomeProofs*( store: BlockStore, tree: ArchivistTree, iter: Iter[int] ): Future[?!void] {.async: (raises: [CancelledError]).} = - without treeCid =? tree.rootCid, err: - return failure(err) + let treeCid = ?tree.rootCid + var items: seq[(Natural, Cid, ArchivistProof)] for i in iter: if i notin 0 ..< tree.leavesCount: return failure( @@ -33,17 +35,13 @@ proc putSomeProofs*( $tree.leavesCount & " leaves" ) - without blkCid =? tree.getLeafCid(i), err: - return failure(err) - - without proof =? tree.getProof(i), err: - return failure(err) - - let res = await store.putCidAndProof(treeCid, i, blkCid, proof) + let + blkCid = ?tree.getLeafCid(i) + proof = ?tree.getProof(i) - if err =? res.errorOption: - return failure(err) + items.add((i.Natural, blkCid, proof)) + ?await store.putCidsAndProofs(treeCid, items) success() proc putSomeProofs*( diff --git a/archivist/streams/storestream.nim b/archivist/streams/storestream.nim index 0b4665c1..2c6bfe1c 100644 --- a/archivist/streams/storestream.nim +++ b/archivist/streams/storestream.nim @@ -7,10 +7,11 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import std/options - {.push raises: [].} +import std/tables +import std/options + import pkg/chronos import pkg/stew/ptrops @@ -71,7 +72,9 @@ proc newLPStreamReadError*(p: ref CatchableError): ref LPStreamReadError = method readOnce*( self: StoreStream, pbytes: pointer, nbytes: int ): Future[int] {.async: (raises: [CancelledError, LPStreamError]).} = - ## Read `nbytes` from current position in the StoreStream into output buffer pointed by `pbytes`. + ## Read `nbytes` from current position in the StoreStream into output + ## buffer pointed by `pbytes`. + ## ## Return how many bytes were actually read before EOF was encountered. ## Raise exception if we are already at EOF. ## @@ -79,47 +82,69 @@ method readOnce*( if self.atEof: raise newLPStreamEOFError() - # The loop iterates over blocks in the StoreStream, - # reading them and copying their data into outbuf - var read = 0 # Bytes read so far, and thus write offset in the outbuf - while read < nbytes and not self.atEof: - # Compute from the current stream position `self.offset` the block num/offset to read - # Compute how many bytes to read from this block + let + blockSize = self.manifest.blockSize.int + treeCid = self.manifest.treeCid + firstBlock = self.offset div blockSize + lastBlock = + min(self.manifest.blocksCount - 1, (self.offset + nbytes - 1) div blockSize) + + # Prefetch all blocks in range as a batch + let indices = (firstBlock .. lastBlock).mapIt(it.Natural) + trace "Requesting indices from store", indices + without blocks =? (await self.store.getBlocks(treeCid, indices)).tryGet.catch, err: + trace "Unable to get blocks from store", err = err.msg + raise newLPStreamReadError(err) + + if blocks.len == 0: + trace "No blocks returned from store!" + raise newLPStreamReadError(newException(IOError, "No blocks returned from store!")) + + # Build a lookup table from block CID to block data for ordered copying + # We copy block by block in index order using single-block getBlock fallback + # if a block is missing from the batch result. + var read = 0 + let blocksMap = blocks.toTable + + for idx in indices: + if self.atEof or read >= nbytes: + break + + if idx notin blocksMap: + break + + without blk =? catch(blocksMap[idx]), err: + trace "Block index not found in returned batch, some blocks failed to retrieve", + err = err.msg + raise newLPStreamReadError(err) + + trace "Read block", cid = blk.cid + let - blockNum = self.offset div self.manifest.blockSize.int - blockOffset = self.offset mod self.manifest.blockSize.int - readBytes = min( - [ - self.size - self.offset, - nbytes - read, - self.manifest.blockSize.int - blockOffset, - ] - ) - address = - BlockAddress(leaf: true, treeCid: self.manifest.treeCid, index: blockNum) - - # Read contents of block `blockNum` - without blk =? (await self.store.getBlock(address)).tryGet.catch, error: - raise newLPStreamReadError(error) + blockOffset = (self.offset + read) mod blockSize + readBytes = + min([self.size - self.offset - read, nbytes - read, blockSize - blockOffset]) + + trace "Read bytes", readBytes + if readBytes <= 0: + break trace "Reading bytes from store stream", - manifestCid = self.manifest.treeCid, + manifestCid = treeCid, numBlocks = self.manifest.blocksCount, - blockNum, + blockNum = idx, blkCid = blk.cid, bytes = readBytes, blockOffset - # Copy `readBytes` bytes starting at `blockOffset` from the block into the outbuf if blk.isEmpty: zeroMem(pbytes.offset(read), readBytes) else: copyMem(pbytes.offset(read), blk.data[blockOffset].unsafeAddr, readBytes) - # Update current positions in the stream and outbuf - self.offset += readBytes read += readBytes + self.offset += read return read method closeImpl*(self: StoreStream) {.async: (raises: []).} = diff --git a/archivist/utils.nim b/archivist/utils.nim index 02d7ce9f..78c8db2d 100644 --- a/archivist/utils.nim +++ b/archivist/utils.nim @@ -10,22 +10,53 @@ {.push raises: [].} -import std/enumerate import std/parseutils import std/options import pkg/chronos +import pkg/stew/bitseqs import ./utils/asyncheapqueue import ./utils/fileutils import ./utils/asynciter import ./utils/safeasynciter -export asyncheapqueue, fileutils, asynciter, safeasynciter, chronos +export asyncheapqueue, fileutils, asynciter, safeasynciter, chronos, bitseqs when defined(posix): import os, posix +type Bytes = seq[byte] + +func combineSafe*(tgt: var BitSeq, src: BitSeq) = + ## OR-combine two BitSeqs that may have different lengths. + ## + + if Bytes(src).len == 0: + return + + if Bytes(tgt).len == 0: + tgt = BitSeq.init(src.len) + for i in 0 ..< src.len: + if src[i]: + tgt.setBit(i) + return + + if tgt.len == src.len: + tgt.combine(src) + return + + let minLen = min(tgt.len, src.len) + + # OR the overlapping portion + for i in 0 ..< minLen: + if src[i]: + tgt.setBit(i) + + # Extend tgt with src's extra bits + for i in minLen ..< src.len: + tgt.add(src[i]) + func divUp*[T: SomeInteger](a, b: T): T = ## Division with result rounded up (rather than truncated as in 'div') assert(b != T(0)) diff --git a/archivist/utils/asynciter.nim b/archivist/utils/asynciter.nim index d87ff67f..67bc7ddb 100644 --- a/archivist/utils/asynciter.nim +++ b/archivist/utils/asynciter.nim @@ -1,18 +1,45 @@ +## AsyncIter[T] - Asynchronous iterator with mandatory disposal +## +## Similar to `Iter[Future[T]]` with addition of methods specific to asynchronous processing. +## +## USAGE CONTRACT: +## 1. Single-consumer: Do NOT call next() concurrently from multiple tasks +## 2. Disposal required: Always call dispose() when done (use defer or try/finally) +## 3. No concurrent dispose: Do NOT call dispose() concurrently or while next() is in-flight +## 4. After dispose: Calling next() will raise an error +## +## Example: +## let iter = createIterator() +## defer: await iter.dispose() +## while not iter.finished: +## let item = await iter.next() +## # process item + import std/sugar import pkg/questionable +import pkg/questionable/results import pkg/chronos import ./iter export iter -## AsyncIter[T] is similar to `Iter[Future[T]]` with addition of methods specific to asynchronous processing -## +type + AsyncDispose* = proc(): Future[void] {.async, gcsafe, closure.} + AsyncIsDisposed* = proc(): bool {.raises: [], gcsafe, closure.} -type AsyncIter*[T] = ref object - finished: bool - next*: GenNext[Future[T]] + AsyncIter*[T] = ref object + finished: bool + next*: GenNext[Future[T]] + disposeImpl: AsyncDispose + disposedImpl: AsyncIsDisposed + +proc defaultAsyncDispose(): Future[void] {.async.} = + discard + +proc defaultAsyncIsDisposed(): bool = + false proc finish*[T](self: AsyncIter[T]): void = self.finished = true @@ -20,6 +47,19 @@ proc finish*[T](self: AsyncIter[T]): void = proc finished*[T](self: AsyncIter[T]): bool = self.finished +proc disposed*[T](self: AsyncIter[T]): bool = + self.disposedImpl() + +proc dispose*[T](self: AsyncIter[T]): Future[void] {.async.} = + ## Dispose the iterator and release any underlying resources. + ## Caller is responsible for calling this when done with the iterator. + ## Idempotent - safe to call multiple times. + ## Sets finished = true to prevent further iteration. + ## Uses noCancel to ensure cleanup completes even if caller is cancelled. + if not self.disposed: + self.finished = true + await noCancel self.disposeImpl() + iterator items*[T](self: AsyncIter[T]): Future[T] = while not self.finished: yield self.next() @@ -42,15 +82,19 @@ proc new*[T]( _: type AsyncIter[T], genNext: GenNext[Future[T]], isFinished: IsFinished, + dispose: AsyncDispose = defaultAsyncDispose, + isDisposed: AsyncIsDisposed = defaultAsyncIsDisposed, finishOnErr: bool = true, ): AsyncIter[T] = ## Creates a new Iter using elements returned by supplier function `genNext`. ## Iter is finished whenever `isFinished` returns true. - ## + ## Caller is responsible for calling `dispose()` when done with the iterator. - var iter = AsyncIter[T]() + var iter = AsyncIter[T](disposeImpl: dispose, disposedImpl: isDisposed) proc next(): Future[T] {.async.} = + if iter.disposed: + raise newException(CatchableError, "AsyncIter is disposed - cannot call next()") if not iter.finished: var item: T try: @@ -78,7 +122,14 @@ proc new*[T]( return iter proc mapAsync*[T, U](iter: Iter[T], fn: Function[T, Future[U]]): AsyncIter[U] = - AsyncIter[U].new(genNext = () => fn(iter.next()), isFinished = () => iter.finished()) + # Chain dispose to underlying sync iterator + AsyncIter[U].new( + genNext = () => fn(iter.next()), + isFinished = () => iter.finished(), + dispose = proc(): Future[void] {.async.} = + iter.dispose(), + isDisposed = () => iter.disposed, + ) proc new*[U, V: Ordinal](_: type AsyncIter[U], slice: HSlice[U, V]): AsyncIter[U] = ## Creates new Iter from a slice @@ -108,17 +159,29 @@ proc empty*[T](_: type AsyncIter[T]): AsyncIter[T] = ## Creates an empty AsyncIter ## + var disposed = false + proc genNext(): Future[T] {.raises: [CatchableError].} = raise newException(CatchableError, "Next item requested from an empty AsyncIter") proc isFinished(): bool = true - AsyncIter[T].new(genNext, isFinished) + proc onDispose(): Future[void] {.async.} = + disposed = true + + proc isDisposed(): bool = + disposed + + AsyncIter[T].new(genNext, isFinished, onDispose, isDisposed) proc map*[T, U](iter: AsyncIter[T], fn: Function[T, Future[U]]): AsyncIter[U] = + # Chain dispose to underlying iterator AsyncIter[U].new( - genNext = () => iter.next().flatMap(fn), isFinished = () => iter.finished + genNext = () => iter.next().flatMap(fn), + isFinished = () => iter.finished, + dispose = () => iter.dispose(), + isDisposed = () => iter.disposed, ) proc mapFilter*[T, U]( @@ -153,7 +216,13 @@ proc mapFilter*[T, U]( nextFutU.isNone await tryFetch() - AsyncIter[U].new(genNext, isFinished) + # Chain dispose to underlying iterator + AsyncIter[U].new( + genNext, + isFinished, + dispose = () => iter.dispose(), + isDisposed = () => iter.disposed, + ) proc filter*[T]( iter: AsyncIter[T], predicate: Function[T, Future[bool]] @@ -164,12 +233,14 @@ proc filter*[T]( else: T.none + # mapFilter already chains dispose to iter await mapFilter[T, T](iter, wrappedPredicate) proc delayBy*[T](iter: AsyncIter[T], d: Duration): AsyncIter[T] = ## Delays emitting each item by given duration ## + # map already chains dispose to iter map[T, T]( iter, proc(t: T): Future[T] {.async.} = diff --git a/archivist/utils/iter.nim b/archivist/utils/iter.nim index 9afd6c12..832fd149 100644 --- a/archivist/utils/iter.nim +++ b/archivist/utils/iter.nim @@ -1,3 +1,18 @@ +## Iter[T] - Synchronous iterator with mandatory disposal +## +## USAGE CONTRACT: +## 1. Single-consumer: Do NOT call next() concurrently from multiple threads/tasks +## 2. Disposal required: Always call dispose() when done (use defer or try/finally) +## 3. No concurrent dispose: Do NOT call dispose() while next() is in-flight +## 4. After dispose: Calling next() will raise an error +## +## Example: +## let iter = createIterator() +## defer: iter.dispose() +## while not iter.finished: +## let item = iter.next() +## # process item + import std/sugar import pkg/questionable @@ -6,11 +21,22 @@ import pkg/questionable/results type Function*[T, U] = proc(fut: T): U {.raises: [CatchableError], gcsafe, closure.} IsFinished* = proc(): bool {.raises: [], gcsafe, closure.} + IsDisposed* = proc(): bool {.raises: [], gcsafe, closure.} + Dispose* = proc() {.raises: [], gcsafe, closure.} GenNext*[T] = proc(): T {.raises: [CatchableError], gcsafe.} Iterator[T] = iterator (): T - Iter*[T] = ref object + + IterObj[T] = object finished: bool next*: GenNext[T] + disposeImpl: Dispose + disposedImpl: IsDisposed + + Iter*[T] = ref IterObj[T] + +# Note: We intentionally don't use =destroy to auto-dispose because closures +# captured in disposeImpl might reference objects that are garbage collected +# before the Iter itself. Callers MUST call dispose() explicitly. proc finish*[T](self: Iter[T]): void = self.finished = true @@ -18,6 +44,18 @@ proc finish*[T](self: Iter[T]): void = proc finished*[T](self: Iter[T]): bool = self.finished +proc disposed*[T](self: Iter[T]): bool = + self.disposedImpl() + +proc dispose*[T](self: Iter[T]) = + ## Dispose the iterator and release any underlying resources. + ## Caller is responsible for calling this when done with the iterator. + ## Idempotent - safe to call multiple times. + ## Sets finished = true to prevent further iteration. + if not self.disposed: + self.finished = true + self.disposeImpl() + iterator items*[T](self: Iter[T]): T = while not self.finished: yield self.next() @@ -32,15 +70,24 @@ proc new*[T]( _: type Iter[T], genNext: GenNext[T], isFinished: IsFinished, + dispose: Dispose, + isDisposed: IsDisposed, finishOnErr: bool = true, ): Iter[T] = ## Creates a new Iter using elements returned by supplier function `genNext`. ## Iter is finished whenever `isFinished` returns true. + ## Caller is responsible for calling `dispose()` when done with the iterator. ## + ## IMPORTANT: dispose and isDisposed callbacks are REQUIRED - passing nil will assert. - var iter = Iter[T]() + doAssert dispose != nil, "dispose callback is required" + doAssert isDisposed != nil, "isDisposed callback is required" + + var iter = Iter[T](disposeImpl: dispose, disposedImpl: isDisposed) proc next(): T {.raises: [CatchableError].} = + if iter.disposed: + raise newException(CatchableError, "Iter is disposed - cannot call next()") if not iter.finished: var item: T try: @@ -66,7 +113,9 @@ proc new*[U, V, S: Ordinal](_: type Iter[U], a: U, b: V, step: S = 1): Iter[U] = ## Creates a new Iter in range a..b with specified step (default 1) ## - var i = a + var + i = a + disposed = false proc genNext(): U = let u = i @@ -76,7 +125,13 @@ proc new*[U, V, S: Ordinal](_: type Iter[U], a: U, b: V, step: S = 1): Iter[U] = proc isFinished(): bool = (step > 0 and i > b) or (step < 0 and i < b) - Iter[U].new(genNext, isFinished) + proc onDispose() = + disposed = true + + proc isDisposed(): bool = + disposed + + Iter[U].new(genNext, isFinished, onDispose, isDisposed) proc new*[U, V: Ordinal](_: type Iter[U], slice: HSlice[U, V]): Iter[U] = ## Creates a new Iter from a slice @@ -93,7 +148,10 @@ proc new*[T](_: type Iter[T], items: seq[T]): Iter[T] = proc new*[T](_: type Iter[T], iter: Iterator[T]): Iter[T] = ## Creates a new Iter from an iterator ## - var nextOrErr: Option[?!T] + var + nextOrErr: Option[?!T] + disposed = false + proc tryNext(): void = nextOrErr = none(?!T) while not iter.finished: @@ -118,23 +176,43 @@ proc new*[T](_: type Iter[T], iter: Iterator[T]): Iter[T] = proc isFinished(): bool = nextOrErr.isNone + proc onDispose() = + disposed = true + + proc isDisposed(): bool = + disposed + tryNext() - Iter[T].new(genNext, isFinished) + Iter[T].new(genNext, isFinished, onDispose, isDisposed) proc empty*[T](_: type Iter[T]): Iter[T] = ## Creates an empty Iter ## + var disposed = false + proc genNext(): T {.raises: [CatchableError].} = raise newException(CatchableError, "Next item requested from an empty Iter") proc isFinished(): bool = true - Iter[T].new(genNext, isFinished) + proc onDispose() = + disposed = true + + proc isDisposed(): bool = + disposed + + Iter[T].new(genNext, isFinished, onDispose, isDisposed) proc map*[T, U](iter: Iter[T], fn: Function[T, U]): Iter[U] = - Iter[U].new(genNext = () => fn(iter.next()), isFinished = () => iter.finished) + # Chain dispose to underlying iterator + Iter[U].new( + genNext = () => fn(iter.next()), + isFinished = () => iter.finished, + dispose = () => iter.dispose(), + isDisposed = () => iter.disposed, + ) proc mapFilter*[T, U](iter: Iter[T], mapPredicate: Function[T, Option[U]]): Iter[U] = var nextUOrErr: Option[?!U] @@ -165,7 +243,13 @@ proc mapFilter*[T, U](iter: Iter[T], mapPredicate: Function[T, Option[U]]): Iter nextUOrErr.isNone tryFetch() - Iter[U].new(genNext, isFinished) + # Chain dispose to underlying iterator + Iter[U].new( + genNext, + isFinished, + dispose = () => iter.dispose(), + isDisposed = () => iter.disposed, + ) proc filter*[T](iter: Iter[T], predicate: Function[T, bool]): Iter[T] = proc wrappedPredicate(t: T): Option[T] = diff --git a/archivist/utils/safeasynciter.nim b/archivist/utils/safeasynciter.nim index cbedb3c5..0c01c4bb 100644 --- a/archivist/utils/safeasynciter.nim +++ b/archivist/utils/safeasynciter.nim @@ -7,6 +7,24 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +## SafeAsyncIter[T] - Exception-safe asynchronous iterator with mandatory disposal +## +## Similar to `AsyncIter[Future[T]]` but does not throw exceptions other than +## CancelledError. It is thus way easier to use with checked exceptions. +## +## USAGE CONTRACT: +## 1. Single-consumer: Do NOT call next() concurrently from multiple tasks +## 2. Disposal required: Always call dispose() when done (use defer or try/finally) +## 3. No concurrent dispose: Do NOT call dispose() concurrently or while next() is in-flight +## 4. After dispose: Calling next() will return failure +## +## Example: +## let iter = createIterator() +## defer: discard await iter.dispose() +## while not iter.finished: +## let item = ?await iter.next() +## # process item + {.push raises: [].} import std/sugar @@ -16,11 +34,6 @@ import pkg/questionable/results import pkg/chronos import ./iter - -## SafeAsyncIter[T] is similar to `AsyncIter[Future[T]]` -## but does not throw exceptions others than CancelledError. -## It is thus way easier to use with checked exceptions -## ## ## Public interface: ## @@ -31,6 +44,8 @@ import ./iter ## - new - to create a new async iterator (SafeAsyncIter) ## - finish - to finish the async iterator ## - finished - to check if the async iterator is finished +## - disposed - to check if the async iterator has been disposed +## - dispose - to dispose the async iterator and release resources ## - next - to get the next item from the async iterator ## - items - to iterate over the async iterator ## - pairs - to iterate over the async iterator and return the index of each item @@ -44,13 +59,23 @@ import ./iter type SafeFunction[T, U] = proc(fut: T): Future[U] {.async: (raises: [CancelledError]), gcsafe, closure.} - SafeIsFinished = proc(): bool {.raises: [], gcsafe, closure.} - SafeGenNext[T] = proc(): Future[T] {.async: (raises: [CancelledError]).} + SafeIsFinished* = proc(): bool {.raises: [], gcsafe, closure.} + SafeIsDisposed* = proc(): bool {.raises: [], gcsafe, closure.} + SafeDispose* = proc(): Future[?!void] {.async: (raises: []), gcsafe, closure.} + SafeGenNext*[T] = proc(): Future[T] {.async: (raises: [CancelledError]).} SafeAsyncIter*[T] = ref object finished: bool nextImpl: proc(iter: SafeAsyncIter[T]): Future[?!T] {.async: (raises: [CancelledError]).} + disposeImpl: SafeDispose + disposedImpl: SafeIsDisposed + +proc defaultSafeDispose(): Future[?!void] {.async: (raises: []).} = + success() + +proc defaultSafeIsDisposed(): bool = + false proc flatMap[T, U]( fut: auto, fn: SafeFunction[?!T, ?!U] @@ -68,18 +93,25 @@ proc flatMap[T, U]( ## SafeAsyncIter public interface methods ######################################################################## +# Forward declarations for procs used inside new() +proc disposed*[T](self: SafeAsyncIter[T]): bool + proc new*[T]( _: type SafeAsyncIter[T], genNext: SafeGenNext[?!T], isFinished: SafeIsFinished, + dispose: SafeDispose = defaultSafeDispose, + isDisposed: SafeIsDisposed = defaultSafeIsDisposed, finishOnErr: bool = true, ): SafeAsyncIter[T] = ## Creates a new Iter using elements returned by supplier function `genNext`. ## Iter is finished whenever `isFinished` returns true. - ## + ## Caller is responsible for calling `dispose()` when done with the iterator. proc next(iter: SafeAsyncIter[T]): Future[?!T] {.async: (raises: [CancelledError]).} = try: + if iter.disposed: + return failure("SafeAsyncIter is disposed - cannot call next()") if not iter.finished: let item = await genNext() if finishOnErr and err =? item.errorOption: @@ -94,7 +126,12 @@ proc new*[T]( iter.finished = true raise err - return SafeAsyncIter[T](nextImpl: next, finished: isFinished()) + return SafeAsyncIter[T]( + nextImpl: next, + finished: isFinished(), + disposeImpl: dispose, + disposedImpl: isDisposed, + ) proc next*[T]( iter: SafeAsyncIter[T] @@ -140,6 +177,21 @@ proc finish*[T](self: SafeAsyncIter[T]): void = proc finished*[T](self: SafeAsyncIter[T]): bool = self.finished +proc disposed*[T](self: SafeAsyncIter[T]): bool = + self.disposedImpl() + +proc dispose*[T](self: SafeAsyncIter[T]): Future[?!void] {.async: (raises: []).} = + ## Dispose the iterator and release any underlying resources. + ## Caller is responsible for calling this when done with the iterator. + ## Idempotent - safe to call multiple times. + ## Sets finished = true to prevent further iteration. + ## Uses noCancel to ensure cleanup completes even if caller is cancelled. + if not self.disposed: + self.finished = true + return await noCancel self.disposeImpl() + + success() + iterator items*[T](self: SafeAsyncIter[T]): auto {.inline.} = while not self.finished: yield self.next() @@ -153,19 +205,27 @@ iterator pairs*[T](self: SafeAsyncIter[T]): auto {.inline.} = proc mapAsync*[T, U]( iter: Iter[T], fn: SafeFunction[T, ?!U], finishOnErr: bool = true ): SafeAsyncIter[U] = + # Chain dispose to underlying sync iterator SafeAsyncIter[U].new( genNext = () => fn(iter.next()), isFinished = () => iter.finished(), + dispose = proc(): Future[?!void] {.async: (raises: []).} = + iter.dispose() + success(), + isDisposed = () => iter.disposed, finishOnErr = finishOnErr, ) proc map*[T, U]( iter: SafeAsyncIter[T], fn: SafeFunction[?!T, ?!U], finishOnErr: bool = true ): SafeAsyncIter[U] = + # Chain dispose to underlying iterator SafeAsyncIter[U].new( genNext = () => iter.next().flatMap(fn), isFinished = () => iter.finished, finishOnErr = finishOnErr, + dispose = () => iter.dispose(), + isDisposed = () => iter.disposed, ) proc mapFilter*[T, U]( @@ -192,7 +252,14 @@ proc mapFilter*[T, U]( nextU.isNone await filter() - SafeAsyncIter[U].new(genNext, isFinished, finishOnErr = finishOnErr) + # Chain dispose to underlying iterator + SafeAsyncIter[U].new( + genNext, + isFinished, + finishOnErr = finishOnErr, + dispose = () => iter.dispose(), + isDisposed = () => iter.disposed, + ) proc filter*[T]( iter: SafeAsyncIter[T], predicate: SafeFunction[?!T, bool], finishOnErr: bool = true @@ -221,6 +288,15 @@ proc delayBy*[T]( finishOnErr = finishOnErr, ) +proc collect*[T]( + iter: SafeAsyncIter[T] +): Future[?!seq[T]] {.async: (raises: [CancelledError]).} = + var res: seq[T] + for item in iter: + res.add(?(await item)) + + success res + proc empty*[T](_: type SafeAsyncIter[T]): SafeAsyncIter[T] = ## Creates an empty SafeAsyncIter ## @@ -231,4 +307,11 @@ proc empty*[T](_: type SafeAsyncIter[T]): SafeAsyncIter[T] = proc isFinished(): bool = true - SafeAsyncIter[T].new(genNext, isFinished) + SafeAsyncIter[T].new( + genNext, + isFinished, + dispose = proc(): Future[?!void] {.async: (raises: []).} = + success(), + isDisposed = proc(): bool = + true, + ) diff --git a/benchmarks/run_benchmarks.nim b/benchmarks/run_benchmarks.nim index 215b00fc..049f373f 100644 --- a/benchmarks/run_benchmarks.nim +++ b/benchmarks/run_benchmarks.nim @@ -3,7 +3,6 @@ import std/[times, os, strutils, terminal] import pkg/questionable import pkg/questionable/results -import pkg/datastore import pkg/archivist/[rng, stores, merkletree, archivisttypes, slots] import pkg/archivist/utils/[json, poseidon2digest] diff --git a/metrics/archivist-grafana-dashboard.json b/metrics/archivist-grafana-dashboard.json index 42f58b30..15b4e02a 100644 --- a/metrics/archivist-grafana-dashboard.json +++ b/metrics/archivist-grafana-dashboard.json @@ -26,10 +26,8 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 13, - "iteration": 1659993037535, + "id": 743, "links": [], - "liveNow": false, "panels": [ { "collapsed": false, @@ -41,13 +39,13 @@ }, "id": 48, "panels": [], - "title": "Main", + "title": "Node", "type": "row" }, { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -57,7 +55,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 }, { "color": "red", @@ -71,7 +69,7 @@ }, "gridPos": { "h": 3, - "w": 12, + "w": 24, "x": 0, "y": 1 }, @@ -81,6 +79,7 @@ "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -88,448 +87,55 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.0.2", + "pluginVersion": "12.1.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "archivist_inflight_discovery{}", + "editorMode": "code", + "expr": "archivist_inflight_discovery{instance=~\"$node\"}", + "legendFormat": "{{instance}}", + "range": true, "refId": "A" } ], "title": "Archivist Inflight Discovery", "type": "stat" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "expr": "nim_gc_heap_instance_occupied_bytes{node=\"${node}\"}", - "interval": "", - "legendFormat": "{{type_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC heap objects #${node}", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": false - }, - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 6, - "x": 0, - "y": 4 - }, - "id": 6, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^process_resident_memory_bytes{instance=\"127.0.0.1:8008\", job=\"nimbus\", node=\"0\"}$/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "expr": "process_resident_memory_bytes{node=\"${node}\"}", - "refId": "A" - } - ], - "title": "RSS mem #${node}", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 6, - "x": 6, - "y": 4 - }, - "id": 8, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^{instance=\"127.0.0.1:8008\", job=\"nimbus\", node=\"0\"}$/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "expr": "rate(node_cpu_seconds_total{job=\"archivist\"}[1m])", - "refId": "A" - } - ], - "title": "CPU usage #${node}", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "RSS", - "yaxis": 2 - }, - { - "alias": "Nim GC mem total", - "yaxis": 2 - }, - { - "alias": "Nim GC mem used", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "expr": "rate(process_cpu_seconds_total{node=\"${node}\"}[2s]) * 100", - "legendFormat": "CPU usage %", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "expr": "process_open_fds{node=\"${node}\"}", - "legendFormat": "open file descriptors", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "expr": "process_resident_memory_bytes{node=\"${node}\"}", - "legendFormat": "RSS", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "expr": "nim_gc_mem_bytes{node=\"${node}\"}", - "legendFormat": "Nim GC mem total", - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "expr": "nim_gc_mem_occupied_bytes{node=\"${node}\"}", - "legendFormat": "Nim GC mem used", - "refId": "G" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "resources #${node}", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 50, - "panels": [], - "title": "libp2p", - "type": "row" - }, { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, + "description": "Basic CPU info", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 40, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -540,7 +146,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -548,13 +154,14 @@ }, "links": [], "mappings": [], + "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": null + "value": 0 }, { "color": "red", @@ -564,645 +171,8648 @@ }, "unit": "short" }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 44, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] }, - "expr": "sum by(type) (libp2p_peers{node=\"${node}\"})", - "interval": "", - "legendFormat": "connected peers", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#1F78C1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "guest" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "irq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "nice" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "softirq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "steal" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "system" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "user" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 4 + }, + "id": 106, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode=\"system\",}[5m])) * 100", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Busy System - {{instance}}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode='user'}[5m])) * 100", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Busy User - {{instance}}", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode='iowait'}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Busy Iowait - {{instance}}", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode=~\".*irq\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Busy IRQs - {{instance}}", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq',}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Busy Other - {{instance}}", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode='idle'}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Idle - {{instance}}", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "CPU Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic memory usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SWAP Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Cache + Buffer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avaliable" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#DEDAF7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 9, + "y": 4 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance)(process_resident_memory_bytes{job=\"archivist-nodes\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RSS - {{instance}}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance)(process_virtual_memory_bytes{job=\"archivist-nodes\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Virtual Memory - {{instance}}", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "Memory Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic network info per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 17, + "y": 4 + }, + "id": 108, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(node_network_receive_bytes_total{}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "recv {{device}}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(node_network_transmit_bytes_total{}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trans {{device}} ", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "RSS" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nim GC mem total" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nim GC mem used" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{instance=~\"$node\"}[2s]) * 100", + "hide": true, + "legendFormat": "CPU usage % - {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "process_open_fds{instance=~\"$node\"}", + "hide": true, + "legendFormat": "open file descriptors - {{instance}} ", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "process_resident_memory_bytes{instance=~\"$node\"}", + "hide": true, + "legendFormat": "RSS - {{instance}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nim_gc_mem_bytes{instance=~\"$node\"}", + "hide": true, + "legendFormat": "Nim GC mem total - {{instance}}", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nim_gc_mem_occupied_bytes{instance=~\"$node\"}", + "hide": false, + "legendFormat": "Nim GC mem used - {{instance}}", + "range": true, + "refId": "G" + } + ], + "title": "Resources - $node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "hidden", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "auto" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nim_gc_mem_occupied_bytes{instance=~\"$node\"}", + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "GC heap objects - $pod", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 97, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows the time each future (async proc) spent occupying the Chronos thread.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 12, + "x": 0, + "y": 231 + }, + "id": 98, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "d32523fb-ef36-4856-b61a-b31e73a39fc2" + }, + "editorMode": "code", + "expr": "chronos_exec_time_with_children_total{instance=~\"$node\"}", + "instant": false, + "legendFormat": "{{proc}} [{{file}}:{{line}}]", + "range": true, + "refId": "A" + } + ], + "title": "Cumulative Chronos Thread Occupancy (exec time)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows the time each future (async proc) spent occupying the Chronos thread over a 60 second rolling window.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 12, + "x": 12, + "y": 231 + }, + "id": 99, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "d32523fb-ef36-4856-b61a-b31e73a39fc2" + }, + "editorMode": "code", + "expr": "increase(chronos_exec_time_with_children_total{instance=~\"$node\"}[60s])", + "instant": false, + "legendFormat": "{{proc}} [{{file}}:{{line}}]", + "range": true, + "refId": "A" + } + ], + "title": "Chronos Thread Occupancy (60s increase)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The maximum time taken by a single call of an async proc. Does not take into account calls that have not yet completed.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 305 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "chronos_single_exec_time_max{instance=~\"$node\"}", + "instant": false, + "legendFormat": "{{proc}} [{{file}}:{{line}}]", + "range": true, + "refId": "A" + } + ], + "title": "Max Chronos Thread Occupancy Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 305 + }, + "id": 102, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "increase(chronos_call_count_total{instance=~\"$node\"}[60s])", + "instant": false, + "legendFormat": "{{proc}} [{{file}}:{{line}}]", + "range": true, + "refId": "A" + } + ], + "title": "Call Count (60s increase)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 319 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "chronos_call_count_total{instance=~\"$node\"}", + "instant": false, + "legendFormat": "{{proc}} [{{file}}:{{line}}]", + "range": true, + "refId": "A" + } + ], + "title": "Cumulative Call Count (Total)", + "type": "timeseries" + } + ], + "title": "Chronos Profiler", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 67, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 309 + }, + "id": 69, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_api_uploads_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Uploads - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 318 + }, + "id": 86, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_api_downloads_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Downloads - $pod", + "type": "timeseries" + } + ], + "title": "Archivist - API Upload/Download", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 70, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (libp2p_peers{instance=~\"$node\"})", + "interval": "", + "legendFormat": "connected peers - {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (libp2p_pubsub_peers{instance=~\"$node\"})", + "interval": "", + "legendFormat": "pubsub peers - {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "libp2p peers - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "LPChannel , In - archivist2-4-5f666765d5-hqcg7" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance,type) (libp2p_open_streams{instance=~\"$node\"})", + "hide": true, + "interval": "", + "legendFormat": "{{type}} - {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (libp2p_open_streams{instance=~\"$node\"})", + "hide": true, + "interval": "", + "legendFormat": "combined - {{instance}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (type, dir, instance) (libp2p_open_streams{instance=~\"$node\"})", + "hide": false, + "interval": "", + "legendFormat": "{{type}} , {{dir}} - {{instance}}", + "range": true, + "refId": "C" + } + ], + "title": "libp2p open streams - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 54, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(sum by (instance) (dht_message_requests_incoming_total{instance=~\"$node\"}))", + "legendFormat": "-> {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum by (instance) (discovery_message_requests_outgoing_total{instance=~\"$node\"})", + "hide": false, + "legendFormat": "<- {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Discovery Requests rate - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 65, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(sum by (instance) (dht_session_lru_cache_hits_total{instance=~\"$node\"}))", + "legendFormat": "hits - {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum by (instance) (discovery_session_lru_cache_misses_total{instance=~\"$node\"})", + "hide": false, + "legendFormat": "misses - {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Discovery LRU - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 111 + }, + "id": 71, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (dht_routing_table_nodes {state=\"\", instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "DHT: Routing table nodes - $pod", + "type": "timeseries" + } + ], + "title": "Archivist - Network Status", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 76, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 371 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_block_exchange_blocks_sent_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Blocks sent - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 371 + }, + "id": 80, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "increase(archivist_block_exchange_blocks_sent_total{instance=~\"$node\"}[60s])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Blocks sent (60s increase)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 379 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_block_exchange_want_have_lists_sent_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "WantHave lists sent - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 379 + }, + "id": 104, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "increase(archivist_block_exchange_want_have_lists_sent_total{instance=~\"$node\"}[60s])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "WantHave lists sent - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 387 + }, + "id": 81, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_block_exchange_want_block_lists_sent_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "WantBlock lists sent - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 387 + }, + "id": 105, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "increase(archivist_block_exchange_want_block_lists_sent_total{instance=~\"$node\"}[60s])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "WantBlock lists sent - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 395 + }, + "id": 82, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_block_exchange_want_have_lists_received_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "WantHave lists received - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 395 + }, + "id": 83, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_block_exchange_want_block_lists_received_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "WantBlock lists received - $pod", + "type": "timeseries" + } + ], + "title": "Archivist - Block Exchange", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 72, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 370 + }, + "id": 73, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_repostore_blocks{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of Blocks - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 379 + }, + "id": 75, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_repostore_bytes_used{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes used - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 388 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_repostore_bytes_reserved{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes reserved - $pod", + "type": "timeseries" + } + ], + "title": "Archivist - Local Storage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 130, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 110, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_tryput_retries_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "tryPut retries", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_tryputatomic_retries_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "tryPutAtomic retries", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_trydelete_retries_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "tryDelete retries", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_trydeleteatomic_retries_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "tryDeleteAtomic retries", + "range": true, + "refId": "D" + } + ], + "title": "KVStore API Retries - rate/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 111, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_tryput_exhausted_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "tryPut exhausted", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_tryputatomic_exhausted_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "tryPutAtomic exhausted", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_trydelete_exhausted_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "tryDelete exhausted", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_trydeleteatomic_exhausted_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "tryDeleteAtomic exhausted", + "range": true, + "refId": "D" + } + ], + "title": "KVStore API Exhausted Retries - rate/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 112, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_put_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "put", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_get_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "get", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_has_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "has", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_delete_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "delete", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_putatomic_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "putAtomic", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_deleteatomic_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "deleteAtomic", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_move_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "moveKeys", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_query_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "query", + "range": true, + "refId": "H" + } + ], + "title": "SQL Operations - rate/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 113, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_put_conflict_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "put conflicts", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_delete_conflict_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "delete conflicts", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_putatomic_conflict_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "putAtomic conflicts", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_putatomic_rollback_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "putAtomic rollbacks", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_deleteatomic_conflict_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "deleteAtomic conflicts", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_deleteatomic_rollback_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "deleteAtomic rollbacks", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_sql_moveatomic_error_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "moveAtomic errors", + "range": true, + "refId": "G" + } + ], + "title": "SQL CAS Conflicts - rate/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 114, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_sql_put_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_sql_put_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(kvstore_sql_put_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "SQL Duration p50/p95/p99 - put", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 115, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_sql_get_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_sql_get_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(kvstore_sql_get_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "SQL Duration p50/p95/p99 - get", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 116, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_sql_putatomic_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_sql_putatomic_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(kvstore_sql_putatomic_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "SQL Duration p50/p95/p99 - putAtomic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 117, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_sql_move_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_sql_move_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(kvstore_sql_move_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "SQL Duration p50/p95/p99 - moveKeys", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 118, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_sql_put_batch_size_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "put p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_sql_put_batch_size_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "put p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_sql_putatomic_batch_size_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "putAtomic p50", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_sql_putatomic_batch_size_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "putAtomic p95", + "range": true, + "refId": "D" + } + ], + "title": "SQL Batch Sizes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 119, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_sql_inflight_put{instance=~\"$node\"}", + "instant": false, + "legendFormat": "put", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_sql_inflight_get{instance=~\"$node\"}", + "instant": false, + "legendFormat": "get", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_sql_inflight_has{instance=~\"$node\"}", + "instant": false, + "legendFormat": "has", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_sql_inflight_delete{instance=~\"$node\"}", + "instant": false, + "legendFormat": "delete", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_sql_inflight_putatomic{instance=~\"$node\"}", + "instant": false, + "legendFormat": "putAtomic", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_sql_inflight_deleteatomic{instance=~\"$node\"}", + "instant": false, + "legendFormat": "deleteAtomic", + "range": true, + "refId": "F" + } + ], + "title": "SQL In-Flight Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 120, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_sql_put_value_bytes_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "put p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_sql_put_value_bytes_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "put p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_sql_get_value_bytes_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "get p50", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_sql_get_value_bytes_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "get p95", + "range": true, + "refId": "D" + } + ], + "title": "SQL Value Sizes p50/p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 121, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_sql_active_iterators{instance=~\"$node\"}", + "instant": false, + "legendFormat": "iterators", + "range": true, + "refId": "A" + } + ], + "title": "SQL Active Iterators", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 122, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_fs_put_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "put", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_fs_get_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "get", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_fs_has_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "has", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_fs_delete_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "delete", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_fs_query_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "query", + "range": true, + "refId": "E" + } + ], + "title": "FS Operations - rate/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 123, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_fs_put_conflict_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "put conflicts", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(kvstore_fs_delete_conflict_total_total{instance=~\"$node\"}[5m])", + "instant": false, + "legendFormat": "delete conflicts", + "range": true, + "refId": "B" + } + ], + "title": "FS CAS Conflicts - rate/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 124, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_fs_put_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_fs_put_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(kvstore_fs_put_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "FS Duration p50/p95/p99 - put", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 125, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_fs_get_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_fs_get_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(kvstore_fs_get_duration_seconds_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "FS Duration p50/p95/p99 - get", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 126, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_fs_put_batch_size_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "put p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_fs_put_batch_size_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "put p95", + "range": true, + "refId": "B" + } + ], + "title": "FS Batch Sizes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 127, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_fs_inflight_put{instance=~\"$node\"}", + "instant": false, + "legendFormat": "put", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_fs_inflight_get{instance=~\"$node\"}", + "instant": false, + "legendFormat": "get", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_fs_inflight_has{instance=~\"$node\"}", + "instant": false, + "legendFormat": "has", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_fs_inflight_delete{instance=~\"$node\"}", + "instant": false, + "legendFormat": "delete", + "range": true, + "refId": "D" + } + ], + "title": "FS In-Flight Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 81 + }, + "id": 128, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_fs_put_value_bytes_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "put p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_fs_put_value_bytes_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "put p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(kvstore_fs_get_value_bytes_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "get p50", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kvstore_fs_get_value_bytes_bucket{instance=~\"$node\"}[5m])) by (le, instance))", + "instant": false, + "legendFormat": "get p95", + "range": true, + "refId": "D" + } + ], + "title": "FS Value Sizes p50/p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 81 + }, + "id": 129, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kvstore_fs_active_iterators{instance=~\"$node\"}", + "instant": false, + "legendFormat": "iterators", + "range": true, + "refId": "A" + } + ], + "title": "FS Active Iterators", + "type": "timeseries" + } + ], + "title": "Archivist - KVStore", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 77, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 187 + }, + "id": 84, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_purchases_pending_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 187 + }, + "id": 85, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "expr": "sum (libp2p_pubsub_peers{node=\"${node}\"})", - "interval": "", - "legendFormat": "pubsub peers", - "refId": "B" + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_purchases_submitted_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Submitted - $pod", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "sum (nbc_peers{node=\"${node}\"})", - "interval": "", - "legendFormat": "nbc peers", - "refId": "C" - } - ], - "title": "libp2p peers #${node}", - "type": "timeseries" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "fieldConfig": { - "defaults": { - "links": [] + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 187 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_purchases_started_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Started - $pod", + "type": "timeseries" }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 15 - }, - "hiddenSeries": false, - "id": 16, - "interval": "", - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "sum by(type) (libp2p_open_streams{node=\"${node}\"})", - "interval": "", - "legendFormat": "{{type}}", - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 187 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_purchases_finished_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Finished - $pod", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "sum (libp2p_open_streams{node=\"${node}\"})", - "interval": "", - "legendFormat": "combined", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 195 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_purchases_failed_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Failed - $pod", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "sum by(type, dir) (libp2p_open_streams{node=\"${node}\"})", - "interval": "", - "legendFormat": "{{type}, {dir}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "libp2p open streams #${node}", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "fieldConfig": { - "defaults": { - "links": [] + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 195 + }, + "id": 90, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_purchases_cancelled_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Cancelled - $pod", + "type": "timeseries" }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 20 - }, - "hiddenSeries": false, - "id": 45, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "sum by(initiator,node)(libp2p_mplex_channels{node=\"${node}\"})", - "interval": "", - "legendFormat": "initiator {{initiator}}", - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 195 + }, + "id": 91, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_purchases_unknown_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Unknown - $pod", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "sum(libp2p_mplex_channels{node=\"${node}\"})", - "interval": "", - "legendFormat": "total", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "libp2p mplex channels #${node}", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 195 + }, + "id": 92, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_purchases_error_total{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Error - $pod", + "type": "timeseries" } ], - "yaxis": { - "align": false - } + "title": "Archivist - Purchases", + "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, + "collapsed": true, "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 20 - }, - "hiddenSeries": false, - "id": 46, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "h": 1, + "w": 24, + "x": 0, + "y": 24 }, - "percentage": false, - "pluginVersion": "9.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "id": 109, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "sort_desc(sum by(peer, initiator)(libp2p_mplex_channels{node=\"${node}\"}))", - "interval": "", - "legendFormat": "peer {{peer}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "libp2p mplex channels per peer #${node}", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 188 + }, + "id": 110, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (archivist_proofs_per_period{instance=~\"$node\"})", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Proofs per period - $pod", + "type": "timeseries" } ], - "yaxis": { - "align": false - } + "title": "Archivist - Proofs", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, - "id": 52, - "panels": [], - "title": "Discovery", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "id": 62, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, - "thresholdsStyle": { - "mode": "off" + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 220 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 54, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (instance) (rate(container_cpu_usage_seconds_total{instance=~\"$node\"}[1m]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU Usage - $pod ", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "discovery_message_requests_incoming_total{}", - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 227 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (instance) (container_processes{instance=~\"$node\",image!=\"\",container!~\"POD|\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Processes - {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (instance) (container_threads{instance=~\"$node\",image!=\"\",container!~\"POD|\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Threads - {{instance}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (instance) (container_sockets{instance=~\"$node\",image!=\"\",container!~\"POD|\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Sockets - {{instance}}", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Processes - $pod ", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "discovery_message_requests_outgoing_total{}", - "hide": false, - "refId": "B" - } - ], - "title": "Discovery Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "_v_qlxkVz" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 291 + }, + "id": 95, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (instance) ((container_cpu_cfs_throttled_periods_total{instance=~\"$node\"} / container_cpu_cfs_periods_total{instance=~\"$node\"}) * 100)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU Throttling - $pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 298 + }, + "id": 63, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - { - "color": "red", - "value": 80 - } - ] - } + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{instance=~\"$node\",image!=\"\",container!~\"POD|\"}) by (instance)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory usage - $pod", + "type": "timeseries" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "discovery_session_lru_cache_hits_total{instance=\"127.0.0.1:8008\", job=\"archivist\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, - "viz": true + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 60, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 305 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (instance) (rate(container_network_receive_bytes_total{instance=~\"$node\"}[1m])) * 8", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "-> {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "- sum by (instance) (rate(container_network_transmit_bytes_total{instance=~\"$node\"}[1m])) * 8", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "<- {{instance}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Network I/O (1m avg) - $pod", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "discovery_session_lru_cache_hits_total{}", - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 312 + }, + "id": 94, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (instance) (rate(container_fs_reads_bytes_total{instance=~\"$node\"}[1m]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Reads {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (instance) (rate(container_fs_writes_bytes_total{instance=~\"$node\"}[1m]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Writes {{instance}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Disk Read/Writes - $pod", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "expr": "discovery_session_lru_cache_misses_total{}", - "hide": false, - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 319 + }, + "id": 96, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "kube_pod_container_status_restarts_total{instance=~\"$node\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Restarts - $pod", + "type": "timeseries" } ], - "title": "Discovery LRU", - "type": "timeseries" + "title": "Pods", + "type": "row" } ], - "refresh": "10s", - "schemaVersion": 36, - "style": "dark", - "tags": [], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [ + "archivist" + ], "templating": { "list": [ { "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" + "text": "VictoriaMetrics", + "value": "${datasource}" + }, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": "All", + "value": [ + "$__all" + ] }, "datasource": { "type": "prometheus", - "uid": "_v_qlxkVz" + "uid": "${datasource}" }, - "definition": "label_values(process_virtual_memory_bytes,node)", - "hide": 0, - "includeAll": false, - "multi": false, + "definition": "label_values(process_resident_memory_bytes,instance)", + "includeAll": true, + "label": "Node", + "multi": true, "name": "node", "options": [], "query": { - "query": "label_values(process_virtual_memory_bytes,node)", - "refId": "Prometheus-node-Variable-Query" + "qryType": 1, + "query": "label_values(process_resident_memory_bytes{job=\"archivist-nodes\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" + }, + { + "current": { + "text": [ + "archivist" + ], + "value": [ + "archivist" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(process_resident_memory_bytes,instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(process_resident_memory_bytes,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(process_resident_memory_bytes,instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "pod", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(process_resident_memory_bytes,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" } ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -1219,8 +8829,7 @@ ] }, "timezone": "", - "title": "Archivist Dashboard", - "uid": "pgeNfj2Wz2b", - "version": 24, - "weekStart": "" + "title": "Archivist node", + "uid": "afaae7b4-5be1-4974-923e-2532131ba614", + "version": 3 } diff --git a/metrics/prometheus.yml b/metrics/prometheus.yml index ca92d46b..af8799b6 100644 --- a/metrics/prometheus.yml +++ b/metrics/prometheus.yml @@ -1,10 +1,39 @@ global: - scrape_interval: 12s + scrape_interval: 10s scrape_configs: - - job_name: "archivist" + - job_name: 'archivist-nodes' static_configs: - - targets: ['127.0.0.1:8008'] - - job_name: "node_exporter" + - targets: ['localhost:10008'] + labels: + node: 'e2e-node1' + namespace: 'archivist' + pod: 'archivist-node1' + - targets: ['localhost:10009'] + labels: + node: 'e2e-node2' + namespace: 'archivist' + pod: 'archivist-node2' + - targets: ['localhost:10010'] + labels: + node: 'e2e-node3' + namespace: 'archivist' + pod: 'archivist-node3' + - targets: ['localhost:10011'] + labels: + node: 'e2e-node4' + namespace: 'archivist' + pod: 'archivist-node4' + - targets: ['localhost:10012'] + labels: + node: 'e2e-node5' + namespace: 'archivist' + pod: 'archivist-node5' + + - job_name: 'node_exporter' static_configs: - - targets: ['127.0.0.1:9100'] + - targets: ['localhost:9100'] + labels: + node: 'e2e-host' + namespace: 'archivist' + pod: 'node-exporter' diff --git a/openapi.yaml b/openapi.yaml index e8484932..3d29082e 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -601,6 +601,8 @@ paths: responses: "204": description: Data was successfully deleted. + "404": + description: Cid was not locally available. "400": description: Invalid CID is specified "500": diff --git a/tests/archivist/blockexchange/discovery/testdiscovery.nim b/tests/archivist/blockexchange/discovery/testdiscovery.nim index 8729cc5c..c3889772 100644 --- a/tests/archivist/blockexchange/discovery/testdiscovery.nim +++ b/tests/archivist/blockexchange/discovery/testdiscovery.nim @@ -3,6 +3,8 @@ import std/sugar import std/tables import pkg/chronos +import pkg/kvstore +import pkg/taskpools import pkg/libp2p/errors @@ -19,7 +21,7 @@ import ../../helpers import ../../helpers/mockdiscovery import ../../examples -asyncchecksuite "Block Advertising and Discovery": +suite "Block Advertising and Discovery": let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256) var @@ -34,13 +36,14 @@ asyncchecksuite "Block Advertising and Discovery": advertiser: Advertiser wallet: WalletRef network: BlockExcNetwork - localStore: CacheStore + localStore: BlockStore engine: BlockExcEngine pendingBlocks: PendingBlocksManager + tp: Taskpool setup: while true: - let chunk = await chunker.getBytes() + let chunk = (await chunker.getBytes()).tryGet() if chunk.len <= 0: break @@ -50,7 +53,13 @@ asyncchecksuite "Block Advertising and Discovery": blockDiscovery = MockDiscovery.new() wallet = WalletRef.example network = BlockExcNetwork.new(switch) - localStore = CacheStore.new(blocks.mapIt(it)) + tp = Taskpool.new(num_threads = 4) + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) + for blk in blocks: + (await localStore.putBlock(blk)).tryGet() peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() @@ -77,6 +86,21 @@ asyncchecksuite "Block Advertising and Discovery": switch.mount(network) + teardown: + if not engine.isNil: + await engine.stop() + + if not switch.isNil: + await switch.stop() + + if not localStore.isNil: + await localStore.close() + + if not discovery.isNil: + await discovery.stop() + + tp.shutdown() + test "Should discover want list": let pendingBlocks = blocks.mapIt(engine.pendingBlocks.getWantHandle(it.cid)) @@ -154,20 +178,23 @@ proc asBlock(m: Manifest): bt.Block = let mdata = m.encode().tryGet() bt.Block.new(data = mdata, codec = ManifestCodec).tryGet() -asyncchecksuite "E2E - Multiple Nodes Discovery": +suite "E2E - Multiple Nodes Discovery": var switch: seq[Switch] blockexc: seq[NetworkStore] manifests: seq[Manifest] mBlocks: seq[bt.Block] trees: seq[ArchivistTree] + tp: Taskpool setup: + tp = Taskpool.new(num_threads = 4) + for _ in 0 ..< 4: let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256) var blocks = newSeq[bt.Block]() while true: - let chunk = await chunker.getBytes() + let chunk = (await chunker.getBytes()).tryGet() if chunk.len <= 0: break @@ -182,7 +209,10 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": blockDiscovery = MockDiscovery.new() wallet = WalletRef.example network = BlockExcNetwork.new(s) - localStore = CacheStore.new() + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() @@ -207,6 +237,18 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": blockexc.add(networkStore) teardown: + for bs in blockexc: + if not bs.engine.isNil: + await bs.engine.stop() + + await bs.close() + + for s in switch: + await s.stop() + + if not tp.isNil: + tp.shutdown() + switch = @[] blockexc = @[] manifests = @[] diff --git a/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim b/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim index 0f89ab25..fe2b0c4e 100644 --- a/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/archivist/blockexchange/discovery/testdiscoveryengine.nim @@ -2,6 +2,8 @@ import std/sequtils import std/tables import pkg/chronos +import pkg/kvstore +import pkg/taskpools import pkg/archivist/rng import pkg/archivist/stores @@ -34,10 +36,12 @@ asyncchecksuite "Test Discovery Engine": blockDiscovery: MockDiscovery pendingBlocks: PendingBlocksManager network: BlockExcNetwork + tp: Taskpool setup: + tp = Taskpool.new(num_threads = 4) while true: - let chunk = await chunker.getBytes() + let chunk = (await chunker.getBytes()).tryGet() if chunk.len <= 0: break @@ -53,9 +57,15 @@ asyncchecksuite "Test Discovery Engine": pendingBlocks = PendingBlocksManager.new() blockDiscovery = MockDiscovery.new() + teardown: + tp.shutdown() + test "Should Query Wants": var - localStore = CacheStore.new() + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) discoveryEngine = DiscoveryEngine.new( localStore, peerStore, @@ -81,7 +91,10 @@ asyncchecksuite "Test Discovery Engine": test "Should queue discovery request": var - localStore = CacheStore.new() + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) discoveryEngine = DiscoveryEngine.new( localStore, peerStore, @@ -106,7 +119,10 @@ asyncchecksuite "Test Discovery Engine": test "Should not request more than minPeersPerBlock": var - localStore = CacheStore.new() + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) minPeers = 2 discoveryEngine = DiscoveryEngine.new( localStore, @@ -149,7 +165,10 @@ asyncchecksuite "Test Discovery Engine": test "Should not request if there is already an inflight discovery request": var - localStore = CacheStore.new() + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) discoveryEngine = DiscoveryEngine.new( localStore, peerStore, diff --git a/tests/archivist/blockexchange/engine/testadvertiser.nim b/tests/archivist/blockexchange/engine/testadvertiser.nim index 53671dec..5ea7ad24 100644 --- a/tests/archivist/blockexchange/engine/testadvertiser.nim +++ b/tests/archivist/blockexchange/engine/testadvertiser.nim @@ -1,6 +1,8 @@ import pkg/chronos import pkg/libp2p/routing_record import pkg/archivistdht/discv5/protocol as discv5 +import pkg/kvstore +import pkg/taskpools import pkg/archivist/blockexchange import pkg/archivist/stores @@ -20,6 +22,7 @@ asyncchecksuite "Advertiser": localStore: BlockStore advertiser: Advertiser advertised: seq[Cid] + tp: Taskpool let manifest = Manifest.new( treeCid = Cid.example, blockSize = 123.NBytes, datasetSize = 234.NBytes @@ -29,7 +32,11 @@ asyncchecksuite "Advertiser": setup: blockDiscovery = MockDiscovery.new() - localStore = CacheStore.new() + tp = Taskpool.new(num_threads = 4) + let + repoDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + localStore = RepoStore.new(repoDs, metaDs) advertised = newSeq[Cid]() blockDiscovery.publishBlockProvideHandler = proc( @@ -43,6 +50,7 @@ asyncchecksuite "Advertiser": teardown: await advertiser.stop() + tp.shutdown() proc waitTillQueueEmpty() {.async.} = check eventually advertiser.advertiseQueue.len == 0 @@ -84,7 +92,11 @@ asyncchecksuite "Advertiser": check manifest.treeCid in advertised test "Should advertise existing manifests and their trees": - let newStore = CacheStore.new([manifestBlk]) + let + newRepoDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + newMetaDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + newStore = RepoStore.new(newRepoDs, newMetaDs) + (await newStore.putBlock(manifestBlk)).tryGet() await advertiser.stop() advertiser = Advertiser.new(newStore, blockDiscovery) diff --git a/tests/archivist/blockexchange/engine/testblockexc.nim b/tests/archivist/blockexchange/engine/testblockexc.nim index 580c6651..8ec6cac5 100644 --- a/tests/archivist/blockexchange/engine/testblockexc.nim +++ b/tests/archivist/blockexchange/engine/testblockexc.nim @@ -15,6 +15,8 @@ import ../../../asynctest import ../../examples import ../../helpers +import ../../helpers/nodeutils + asyncchecksuite "NetworkStore engine - 2 nodes": var nodeCmps1, nodeCmps2: NodesComponents @@ -24,8 +26,8 @@ asyncchecksuite "NetworkStore engine - 2 nodes": pendingBlocks1, pendingBlocks2: seq[BlockHandle] setup: - blocks1 = await makeRandomBlocks(datasetSize = 2048, blockSize = 256'nb) - blocks2 = await makeRandomBlocks(datasetSize = 2048, blockSize = 256'nb) + blocks1 = (await makeRandomBlocks(datasetSize = 2048, blockSize = 256'nb)).tryGet + blocks2 = (await makeRandomBlocks(datasetSize = 2048, blockSize = 256'nb)).tryGet nodeCmps1 = generateNodes(1, blocks1).components[0] nodeCmps2 = generateNodes(1, blocks2).components[0] @@ -151,7 +153,7 @@ asyncchecksuite "NetworkStore - multiple nodes": blocks: seq[bt.Block] setup: - blocks = await makeRandomBlocks(datasetSize = 4096, blockSize = 256'nb) + blocks = (await makeRandomBlocks(datasetSize = 4096, blockSize = 256'nb)).tryGet nodes = generateNodes(5) for e in nodes: await e.engine.start() diff --git a/tests/archivist/blockexchange/engine/testengine.nim b/tests/archivist/blockexchange/engine/testengine.nim index 2fe312c0..0d2378ba 100644 --- a/tests/archivist/blockexchange/engine/testengine.nim +++ b/tests/archivist/blockexchange/engine/testengine.nim @@ -6,14 +6,20 @@ import pkg/stew/byteutils import pkg/chronos import pkg/libp2p/errors import pkg/libp2p/routing_record +import pkg/stew/bitseqs import pkg/archivistdht/discv5/protocol as discv5 +import pkg/kvstore +import pkg/taskpools + import pkg/archivist/rng import pkg/archivist/blockexchange import pkg/archivist/stores import pkg/archivist/chunker import pkg/archivist/discovery import pkg/archivist/blocktype +import pkg/archivist/manifest +import pkg/archivist/merkletree import pkg/archivist/utils/asyncheapqueue import ../../../asynctest @@ -37,8 +43,13 @@ asyncchecksuite "NetworkStore engine basic": pendingBlocks: PendingBlocksManager blocks: seq[Block] done: Future[void] + manifest: Manifest + tree: ArchivistTree + treeCid: Cid + tp: Taskpool setup: + tp = Taskpool.new(num_threads = 4) rng = Rng.instance() seckey = PrivateKey.random(rng[]).tryGet() peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet() @@ -49,12 +60,14 @@ asyncchecksuite "NetworkStore engine basic": pendingBlocks = PendingBlocksManager.new() while true: - let chunk = await chunker.getBytes() + let chunk = (await chunker.getBytes()).tryGet() if chunk.len <= 0: break blocks.add(Block.new(chunk).tryGet()) + (manifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() done = newFuture[void]() test "Should send want list to new peers": @@ -72,7 +85,15 @@ asyncchecksuite "NetworkStore engine basic": let network = BlockExcNetwork(request: BlockExcRequest(sendWantList: sendWantList)) - localStore = CacheStore.new(blocks.mapIt(it)) + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) + + # Store blocks with overlay context + (await localStore.storeBlocksWithOverlay(treeCid, blocks, tree)).tryGet() + + let discovery = DiscoveryEngine.new( localStore, peerStore, network, blockDiscovery, pendingBlocks ) @@ -83,6 +104,7 @@ asyncchecksuite "NetworkStore engine basic": for b in blocks: discard engine.pendingBlocks.getWantHandle(b.cid) + await engine.setupPeer(peerId) await done.wait(100.millis) @@ -99,7 +121,10 @@ asyncchecksuite "NetworkStore engine basic": let network = BlockExcNetwork(request: BlockExcRequest(sendAccount: sendAccount)) - localStore = CacheStore.new() + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) discovery = DiscoveryEngine.new( localStore, peerStore, network, blockDiscovery, pendingBlocks ) @@ -115,6 +140,9 @@ asyncchecksuite "NetworkStore engine basic": await done.wait(100.millis) + teardown: + tp.shutdown() + asyncchecksuite "NetworkStore engine handlers": var rng: Rng @@ -130,20 +158,26 @@ asyncchecksuite "NetworkStore engine handlers": discovery: DiscoveryEngine advertiser: Advertiser peerCtx: BlockExcPeerCtx - localStore: BlockStore + localStore: RepoStore blocks: seq[Block] + manifest: Manifest + tree: ArchivistTree + treeCid: Cid + tp: Taskpool setup: rng = Rng.instance() chunker = RandomChunker.new(rng, size = 1024'nb, chunkSize = 256'nb) while true: - let chunk = await chunker.getBytes() + let chunk = (await chunker.getBytes()).tryGet() if chunk.len <= 0: break blocks.add(Block.new(chunk).tryGet()) + (manifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() seckey = PrivateKey.random(rng[]).tryGet() peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet() wallet = WalletRef.example @@ -151,7 +185,11 @@ asyncchecksuite "NetworkStore engine handlers": peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() - localStore = CacheStore.new() + tp = Taskpool.new(num_threads = 4) + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) network = BlockExcNetwork() discovery = @@ -166,6 +204,9 @@ asyncchecksuite "NetworkStore engine handlers": peerCtx = BlockExcPeerCtx(id: peerId) engine.peers.add(peerCtx) + teardown: + tp.shutdown() + test "Should schedule block requests": let wantList = makeWantList(blocks.mapIt(it.cid), wantType = WantType.WantBlock) # only `wantBlock` are stored in `peerWants` @@ -194,7 +235,8 @@ asyncchecksuite "NetworkStore engine handlers": engine.network = BlockExcNetwork(request: BlockExcRequest(sendPresence: sendPresence)) - await allFuturesThrowing(allFinished(blocks.mapIt(localStore.putBlock(it)))) + # Store blocks with tree context + (await localStore.storeBlocksWithOverlay(treeCid, blocks, tree)).tryGet() await engine.wantListHandler(peerId, wantList) await done @@ -240,13 +282,47 @@ asyncchecksuite "NetworkStore engine handlers": engine.network = BlockExcNetwork(request: BlockExcRequest(sendPresence: sendPresence)) - (await engine.localStore.putBlock(blocks[0])).tryGet() - (await engine.localStore.putBlock(blocks[1])).tryGet() + # Store first two blocks with tree context + (await localStore.storeBlocksWithOverlay(treeCid, blocks, tree, @[0, 1])).tryGet() + await engine.wantListHandler(peerId, wantList) await done - test "Should store blocks in local store": + test "Should store leaf blocks in local store": + # Create overlay and store blocks with tree context + let indices = toSeq(0 ..< blocks.len) + (await localStore.storeBlocksWithOverlay(treeCid, blocks, tree)).tryGet() + + # Create pending handles for leaf addresses + let pending = + indices.mapIt(engine.pendingBlocks.getWantHandle(BlockAddress.init(treeCid, it))) + + # Create leaf deliveries with proofs + let blocksDelivery = indices.mapIt( + BlockDelivery( + blk: blocks[it], + address: BlockAddress.init(treeCid, it), + proof: tree.getProof(it).tryGet().some, + ) + ) + + # Install NOP for want list cancellations so they don't cause a crash + engine.network = BlockExcNetwork( + request: BlockExcRequest(sendWantCancellations: NopSendWantCancellationsProc) + ) + + await engine.blocksDeliveryHandler(peerId, blocksDelivery) + let resolved = await allFinished(pending) + check resolved.mapIt(it.read) == blocks + + # Verify blocks are persisted with tree-aware hasBlock + for i in indices: + let present = await engine.localStore.hasBlock(treeCid, i) + check present.tryGet() + + test "Should resolve non-leaf block deliveries without persistence": + # Non-leaf deliveries are validated and resolved but NOT persisted let pending = blocks.mapIt(engine.pendingBlocks.getWantHandle(it.cid)) let blocksDelivery = blocks.mapIt(BlockDelivery(blk: it, address: it.address)) @@ -259,6 +335,8 @@ asyncchecksuite "NetworkStore engine handlers": await engine.blocksDeliveryHandler(peerId, blocksDelivery) let resolved = await allFinished(pending) check resolved.mapIt(it.read) == blocks + + # Non-leaf deliveries are NOT persisted - hasBlock should fail for b in blocks: let present = await engine.localStore.hasBlock(b.cid) check present.tryGet() @@ -378,13 +456,14 @@ asyncchecksuite "Block Download": peerCtx: BlockExcPeerCtx localStore: BlockStore blocks: seq[Block] + tp: Taskpool setup: rng = Rng.instance() chunker = RandomChunker.new(rng, size = 1024'nb, chunkSize = 256'nb) while true: - let chunk = await chunker.getBytes() + let chunk = (await chunker.getBytes()).tryGet() if chunk.len <= 0: break @@ -397,7 +476,11 @@ asyncchecksuite "Block Download": peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() - localStore = CacheStore.new() + tp = Taskpool.new(num_threads = 4) + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) network = BlockExcNetwork() discovery = @@ -412,6 +495,9 @@ asyncchecksuite "Block Download": peerCtx = BlockExcPeerCtx(id: peerId) engine.peers.add(peerCtx) + teardown: + tp.shutdown() + test "Should exhaust retries": var retries = 2 @@ -534,22 +620,28 @@ asyncchecksuite "Task Handler": engine: BlockExcEngine discovery: DiscoveryEngine advertiser: Advertiser - localStore: BlockStore + localStore: RepoStore + tp: Taskpool peersCtx: seq[BlockExcPeerCtx] peers: seq[PeerId] blocks: seq[Block] + manifest: Manifest + tree: ArchivistTree + treeCid: Cid setup: rng = Rng.instance() chunker = RandomChunker.new(rng, size = 1024, chunkSize = 256'nb) while true: - let chunk = await chunker.getBytes() + let chunk = (await chunker.getBytes()).tryGet() if chunk.len <= 0: break blocks.add(Block.new(chunk).tryGet()) + (manifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() seckey = PrivateKey.random(rng[]).tryGet() peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet() wallet = WalletRef.example @@ -557,7 +649,11 @@ asyncchecksuite "Task Handler": peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() - localStore = CacheStore.new() + tp = Taskpool.new(num_threads = 4) + localStore = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) network = BlockExcNetwork() discovery = @@ -579,6 +675,9 @@ asyncchecksuite "Task Handler": engine.pricing = Pricing.example.some + teardown: + tp.shutdown() + test "Should send want-blocks in priority order": proc sendBlocksDelivery( id: PeerId, blocksDelivery: seq[BlockDelivery] @@ -588,8 +687,9 @@ asyncchecksuite "Task Handler": blocksDelivery[1].address == blocks[0].address blocksDelivery[0].address == blocks[1].address - for blk in blocks: - (await engine.localStore.putBlock(blk)).tryGet() + # Store blocks with tree context + (await localStore.storeBlocksWithOverlay(treeCid, blocks, tree)).tryGet() + engine.network.request.sendBlocksDelivery = sendBlocksDelivery # second block to send by priority @@ -622,8 +722,9 @@ asyncchecksuite "Task Handler": ) {.async: (raises: [CancelledError]).} = check peersCtx[0].peerWants[0].inFlight - for blk in blocks: - (await engine.localStore.putBlock(blk)).tryGet() + # Store blocks with tree context + (await localStore.storeBlocksWithOverlay(treeCid, blocks, tree)).tryGet() + engine.network.request.sendBlocksDelivery = sendBlocksDelivery peersCtx[0].peerWants.add( @@ -668,8 +769,9 @@ asyncchecksuite "Task Handler": Presence(address: missing[0].address, have: false), ] - for blk in blocks: - (await engine.localStore.putBlock(blk)).tryGet() + # Store blocks with tree context + (await localStore.storeBlocksWithOverlay(treeCid, blocks, tree)).tryGet() + engine.network.request.sendPresence = sendPresence # have block diff --git a/tests/archivist/blockexchange/testnetwork.nim b/tests/archivist/blockexchange/testnetwork.nim index b63d0718..c5c07f5a 100644 --- a/tests/archivist/blockexchange/testnetwork.nim +++ b/tests/archivist/blockexchange/testnetwork.nim @@ -31,7 +31,7 @@ asyncchecksuite "Network - Handlers": setup: while true: - let chunk = await chunker.getBytes() + let chunk = (await chunker.getBytes()).tryGet() if chunk.len <= 0: break @@ -143,7 +143,7 @@ asyncchecksuite "Network - Senders": setup: while true: - let chunk = await chunker.getBytes() + let chunk = (await chunker.getBytes()).tryGet() if chunk.len <= 0: break diff --git a/tests/archivist/helpers.nim b/tests/archivist/helpers.nim index 4a51bd5c..9aeae66c 100644 --- a/tests/archivist/helpers.nim +++ b/tests/archivist/helpers.nim @@ -3,6 +3,7 @@ import std/sequtils import pkg/chronos import pkg/libp2p import pkg/libp2p/varint +import pkg/stew/bitseqs import pkg/archivist/blocktype import pkg/archivist/stores import pkg/archivist/manifest @@ -11,14 +12,13 @@ import pkg/archivist/blockexchange import pkg/archivist/rng import pkg/archivist/utils -import ./helpers/nodeutils import ./helpers/randomchunker import ./helpers/mockchunker import ./helpers/mockdiscovery import ./helpers/always import ../checktest -export randomchunker, nodeutils, mockdiscovery, mockchunker, always, checktest, manifest +export randomchunker, mockdiscovery, mockchunker, always, checktest, manifest export libp2p except setup, eventually @@ -85,43 +85,51 @@ proc makeWantList*( ) proc storeDataGetManifest*( - store: BlockStore, blocks: seq[Block] -): Future[Manifest] {.async.} = - for blk in blocks: - (await store.putBlock(blk)).tryGet() + store: RepoStore, blocks: seq[Block] +): Future[?!Manifest] {.async: (raises: [CancelledError]).} = + let tmpTreeCid = ?await store.createTmpOverlay() + + for i, blk in blocks: + ?await store.putBlock(tmpTreeCid, blk, i) let - (manifest, tree) = makeManifestAndTree(blocks).tryGet() - treeCid = tree.rootCid.tryGet() + (manifest, tree) = ?makeManifestAndTree(blocks) + treeCid = ?tree.rootCid + + ?await store.finalizeOverlay(tmpTreeCid, treeCid) for i in 0 ..< tree.leavesCount: - let proof = tree.getProof(i).tryGet() - (await store.putCidAndProof(treeCid, i, blocks[i].cid, proof)).tryGet() + let proof = ?tree.getProof(i) + ?await store.putCidAndProof(treeCid, i, blocks[i].cid, proof) - return manifest + success manifest proc storeDataGetManifest*( - store: BlockStore, chunker: Chunker -): Future[Manifest] {.async.} = + store: RepoStore, chunker: Chunker +): Future[?!Manifest] {.async: (raises: [CancelledError]).} = var blocks = newSeq[Block]() - while (let chunk = await chunker.getBytes(); chunk.len > 0): - blocks.add(Block.new(chunk).tryGet()) + while (let chunk = ?await chunker.getBytes(); chunk.len > 0): + blocks.add(?Block.new(chunk)) - return await storeDataGetManifest(store, blocks) + await storeDataGetManifest(store, blocks) proc makeRandomBlocks*( datasetSize: int, blockSize: NBytes -): Future[seq[Block]] {.async.} = - var chunker = - RandomChunker.new(Rng.instance(), size = datasetSize, chunkSize = blockSize) +): Future[?!seq[Block]] {.async.} = + var + chunker = + RandomChunker.new(Rng.instance(), size = datasetSize, chunkSize = blockSize) + blocks: seq[Block] while true: - let chunk = await chunker.getBytes() + let chunk = ?await chunker.getBytes() if chunk.len <= 0: break - result.add(Block.new(chunk).tryGet()) + blocks.add(Block.new(chunk).tryGet()) + + success blocks proc corruptBlocks*( store: BlockStore, manifest: Manifest, blks, bytes: int @@ -148,3 +156,60 @@ proc corruptBlocks*( bytePos.add(ii) blk.data[ii] = byte 0 return pos + +proc makeBitSeq*(len: int, setBits: seq[int] = @[]): BitSeq = + ## Create a BitSeq with specified bits set. + ## If setBits is empty, all bits are set. + ## + var bits = BitSeq.init(len) + if setBits.len == 0: + for i in 0 ..< len: + bits.setBit(i) + else: + for i in setBits: + if i < len: + bits.setBit(i) + bits + +proc storeBlocksWithOverlay*( + store: RepoStore, + treeCid: Cid, + blocks: seq[Block], + tree: ArchivistTree, + indices: seq[int] = @[], + status: ?OverlayStatus = OverlayStatus.Completed.some, +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Store blocks with overlay context. + ## Creates overlay, stores blocks with proofs. + ## If indices is empty, all blocks are stored. + ## + let + idx = + if indices.len == 0: + toSeq(0 ..< blocks.len) + else: + indices + bits = makeBitSeq(blocks.len, idx) + + ?await store.putOverlay(treeCid, status, bits) + + var items: seq[(Block, Natural, ArchivistProof)] + for i in idx: + let proof = ?tree.getProof(i) + items.add((blocks[i], i.Natural, proof)) + + ?await store.putBlocks(treeCid, items) + + success() + +proc storeBlocksWithOverlay*( + store: RepoStore, + treeCid: Cid, + blocks: seq[Block], + tree: ArchivistTree, + indices: openArray[int], + status: ?OverlayStatus = OverlayStatus.Completed.some, +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Store blocks with overlay context (openArray variant). + ## + await storeBlocksWithOverlay(store, treeCid, blocks, tree, @indices, status) diff --git a/tests/archivist/helpers/mockchunker.nim b/tests/archivist/helpers/mockchunker.nim index 23a3b2ed..e4f6b7a7 100644 --- a/tests/archivist/helpers/mockchunker.nim +++ b/tests/archivist/helpers/mockchunker.nim @@ -1,4 +1,5 @@ import pkg/chronos +import pkg/questionable/results import pkg/archivist/chunker export chunker @@ -21,9 +22,9 @@ proc new*( var consumed = 0 proc reader( data: ChunkBuffer, len: int - ): Future[int] {.gcsafe, async: (raises: [ChunkerError, CancelledError]).} = + ): Future[?!int] {.gcsafe, async: (raises: [CancelledError]).} = if consumed >= dataset.len: - return 0 + return success 0 var read = 0 while read < len and read < chunkSize.int and (consumed + read) < dataset.len: @@ -31,6 +32,6 @@ proc new*( read.inc consumed += read - return read + return success read Chunker.new(reader = reader, pad = pad, chunkSize = chunkSize) diff --git a/tests/archivist/helpers/mockrepostore.nim b/tests/archivist/helpers/mockrepostore.nim deleted file mode 100644 index 422b0cef..00000000 --- a/tests/archivist/helpers/mockrepostore.nim +++ /dev/null @@ -1,54 +0,0 @@ -## Copyright (c) 2025 Archivist Authors -## Copyright (c) 2023 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import std/sequtils -import pkg/chronos -import pkg/libp2p -import pkg/questionable -import pkg/questionable/results - -import pkg/archivist/stores/repostore -import pkg/archivist/utils/asynciter -import pkg/archivist/utils/safeasynciter - -type MockRepoStore* = ref object of RepoStore - delBlockCids*: seq[Cid] - getBeMaxNumber*: int - getBeOffset*: int - - testBlockExpirations*: seq[BlockExpiration] - -method delBlock*( - self: MockRepoStore, cid: Cid -): Future[?!void] {.async: (raises: [CancelledError]).} = - self.delBlockCids.add(cid) - self.testBlockExpirations = self.testBlockExpirations.filterIt(it.cid != cid) - return success() - -method getBlockExpirations*( - self: MockRepoStore, maxNumber: int, offset: int -): Future[?!SafeAsyncIter[BlockExpiration]] {.async: (raises: [CancelledError]).} = - self.getBeMaxNumber = maxNumber - self.getBeOffset = offset - - let - testBlockExpirationsCpy = @(self.testBlockExpirations) - limit = min(offset + maxNumber, len(testBlockExpirationsCpy)) - - let - iter1 = SafeAsyncIter[int].new(offset ..< limit) - iter2 = map[int, BlockExpiration]( - iter1, - proc(i: ?!int): Future[?!BlockExpiration] {.async: (raises: [CancelledError]).} = - if i =? i: - return success(testBlockExpirationsCpy[i]) - return failure("Unexpected error!"), - ) - - success(iter2) diff --git a/tests/archivist/helpers/nodeutils.nim b/tests/archivist/helpers/nodeutils.nim index 38d72394..4772cb38 100644 --- a/tests/archivist/helpers/nodeutils.nim +++ b/tests/archivist/helpers/nodeutils.nim @@ -2,6 +2,7 @@ import std/sequtils import pkg/chronos import pkg/taskpools +import pkg/kvstore import pkg/libp2p import pkg/libp2p/errors @@ -39,14 +40,13 @@ type blockDiscovery*: Discovery wallet*: WalletRef network*: BlockExcNetwork - localStore*: BlockStore + localStore*: RepoStore peerStore*: PeerCtxStore pendingBlocks*: PendingBlocksManager discovery*: DiscoveryEngine engine*: BlockExcEngine networkStore*: NetworkStore node*: ArchivistNodeRef = nil - tempDbs*: seq[TempLevelDb] = @[] NodesCluster* = ref object components*: seq[NodesComponents] @@ -66,7 +66,7 @@ converter toTuple*( blockDiscovery: Discovery, wallet: WalletRef, network: BlockExcNetwork, - localStore: BlockStore, + localStore: RepoStore, peerStore: PeerCtxStore, pendingBlocks: PendingBlocksManager, discovery: DiscoveryEngine, @@ -84,7 +84,7 @@ converter toComponents*(cluster: NodesCluster): seq[NodesComponents] = proc nodes*(cluster: NodesCluster): seq[ArchivistNodeRef] = cluster.components.filterIt(it.node != nil).mapIt(it.node) -proc localStores*(cluster: NodesCluster): seq[BlockStore] = +proc localStores*(cluster: NodesCluster): seq[RepoStore] = cluster.components.mapIt(it.localStore) proc switches*(cluster: NodesCluster): seq[Switch] = @@ -136,15 +136,13 @@ proc generateNodes*( peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() - let (localStore, tempDbs, blockDiscovery) = + let (localStore, blockDiscovery) = if config.useRepoStore: let - bdStore = TempLevelDb.new() - repoStore = TempLevelDb.new() - mdStore = TempLevelDb.new() - store = - RepoStore.new(repoStore.newDb(), mdStore.newDb(), clock = SystemClock.new()) - blockDiscoveryStore = bdStore.newDb() + repoDs = SQLiteKVStore.new(SqliteMemory, taskpool).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, taskpool).tryGet() + store = RepoStore.new(repoDs, metaDs, clock = SystemClock.new()) + blockDiscoveryStore = SQLiteKVStore.new(SqliteMemory, taskpool).tryGet() discovery = Discovery.new( switch.peerInfo.privateKey, announceAddrs = @[listenAddr], @@ -153,13 +151,21 @@ proc generateNodes*( bootstrapNodes = bootstrapNodes, ) waitFor store.start() - (store.BlockStore, @[bdStore, repoStore, mdStore], discovery) + (store, discovery) else: let - store = CacheStore.new(blocks.mapIt(it)) - discovery = - Discovery.new(switch.peerInfo.privateKey, announceAddrs = @[listenAddr]) - (store.BlockStore, newSeq[TempLevelDb](), discovery) + repoDs = SQLiteKVStore.new(SqliteMemory, taskpool).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, taskpool).tryGet() + store = RepoStore.new(repoDs, metaDs) + discoveryDs = SQLiteKVStore.new(SqliteMemory, taskpool).tryGet() + discovery = Discovery.new( + switch.peerInfo.privateKey, + announceAddrs = @[listenAddr], + store = discoveryDs, + ) + for blk in blocks: + (waitFor store.putBlock(blk)).tryGet() + (store, discovery) let discovery = DiscoveryEngine.new( @@ -178,6 +184,7 @@ proc generateNodes*( let fullNode = ArchivistNodeRef.new( switch = switch, networkStore = networkStore, + repoStore = localStore, engine = engine, prover = Prover.none, discovery = blockDiscovery, @@ -212,7 +219,6 @@ proc generateNodes*( engine: engine, networkStore: networkStore, node: node, - tempDbs: tempDbs, ) components.add(nodeComponent) @@ -240,15 +246,14 @@ proc connectNodes*(cluster: NodesCluster) {.async.} = proc cleanup*(cluster: NodesCluster) {.async.} = for component in cluster.components: if component.node != nil: - await component.node.switch.stop() - await component.node.stop() - - for component in cluster.components: - for db in component.tempDbs: - await db.destroyDb() + try: + await component.node.stop() + await component.node.switch.stop() + except CatchableError: + discard for component in cluster.components: - if component.tempDbs.len > 0: - await RepoStore(component.localStore).stop() + if not component.localStore.isNil: + await component.localStore.close() cluster.taskpool.shutdown() diff --git a/tests/archivist/helpers/randomchunker.nim b/tests/archivist/helpers/randomchunker.nim index 55e8d1ff..76b6d40b 100644 --- a/tests/archivist/helpers/randomchunker.nim +++ b/tests/archivist/helpers/randomchunker.nim @@ -26,11 +26,11 @@ proc new*( var consumed = 0 proc reader( data: ChunkBuffer, len: int - ): Future[int] {.async: (raises: [ChunkerError, CancelledError]), gcsafe.} = + ): Future[?!int] {.async: (raises: [CancelledError]), gcsafe.} = var alpha = toSeq(byte('A') .. byte('z')) if consumed >= size: - return 0 + return success 0 var read = 0 while read < len and (pad or read < size - consumed): @@ -43,6 +43,6 @@ proc new*( read.inc consumed += read - return read + success read Chunker.new(reader = reader, pad = pad, chunkSize = chunkSize) diff --git a/tests/archivist/marketplace/availability/teststore.nim b/tests/archivist/marketplace/availability/teststore.nim index 4f59d8d9..c33af39f 100644 --- a/tests/archivist/marketplace/availability/teststore.nim +++ b/tests/archivist/marketplace/availability/teststore.nim @@ -1,22 +1,24 @@ import pkg/questionable/results -import pkg/datastore/typedds +import pkg/kvstore +import pkg/taskpools import archivist/marketplace/availability/store import archivist/marketplace/availability/terms import ../../../asynctest -import ../../../helpers/templeveldb import ./examples suite "availability store": var store: AvailabilityStore - var database: TempLevelDb + var database: KVStore + var tp: Taskpool setup: - database = TempLevelDb.new() - let datastore = TypedDatastore.init(database.newDb()) - store = AvailabilityStore.new(datastore) + tp = Taskpool.new(num_threads = 4) + database = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + store = AvailabilityStore.new(database) teardown: - await database.destroyDb() + (await database.close()).tryGet() + tp.shutdown() test "cannot load when there's are no terms saved": let loaded = await store.load() diff --git a/tests/archivist/marketplace/sales/mockstorage.nim b/tests/archivist/marketplace/sales/mockstorage.nim index cf756263..4981fa2d 100644 --- a/tests/archivist/marketplace/sales/mockstorage.nim +++ b/tests/archivist/marketplace/sales/mockstorage.nim @@ -10,15 +10,18 @@ type MockStorage* = ref object of StorageInterface storeSlotResult: ?!void proveSlotResult: ?!Groth16Proof updateSlotExpiryResult: ?!void + slotFailedResult: ?!void storeSlotCalls: seq[(Cid, uint64, uint64, StorageTimestamp, bool)] proveSlotCalls: seq[(Cid, uint64, ProofChallenge)] updateSlotExpiryCalls: seq[(Cid, uint64, StorageTimestamp)] + slotFailedCalls: seq[(Cid, uint64)] proc new*(_: type MockStorage): MockStorage = MockStorage( storeSlotResult: success(), proveSlotResult: success(Groth16Proof.example), updateSlotExpiryResult: success(), + slotFailedResult: success(), ) func `available=`*(mock: MockStorage, value: uint64) = @@ -33,6 +36,9 @@ func `proveSlotResult=`*(mock: MockStorage, value: ?!Groth16Proof) = func `updateSlotExpiryResult=`*(mock: MockStorage, value: ?!void) = mock.updateSlotExpiryResult = value +func `slotFailedResult=`*(mock: MockStorage, value: ?!void) = + mock.slotFailedResult = value + func storeSlotCalls*( mock: MockStorage ): seq[(Cid, uint64, uint64, StorageTimestamp, bool)] = @@ -44,6 +50,9 @@ func proveSlotCalls*(mock: MockStorage): seq[(Cid, uint64, ProofChallenge)] = func updateSlotExpiryCalls*(mock: MockStorage): seq[(Cid, uint64, StorageTimestamp)] = mock.updateSlotExpiryCalls +func slotFailedCalls*(mock: MockStorage): seq[(Cid, uint64)] = + mock.slotFailedCalls + method available*(mock: MockStorage): uint64 {.gcsafe, raises: [].} = mock.available @@ -69,3 +78,9 @@ method updateSlotExpiry*( ): Future[?!void] {.async: (raises: [CancelledError]).} = mock.updateSlotExpiryCalls.add((cid, slotIndex, expiry)) mock.updateSlotExpiryResult + +method deleteSlot*( + mock: MockStorage, cid: Cid, slotIndex: uint64 +): Future[?!void] {.async: (raises: [CancelledError]).} = + mock.slotFailedCalls.add((cid, slotIndex)) + mock.slotFailedResult diff --git a/tests/archivist/marketplace/sales/testsales.nim b/tests/archivist/marketplace/sales/testsales.nim index 27b19fd0..2e732500 100644 --- a/tests/archivist/marketplace/sales/testsales.nim +++ b/tests/archivist/marketplace/sales/testsales.nim @@ -2,7 +2,8 @@ import std/sequtils import std/sugar import std/times import pkg/chronos -import pkg/datastore/typedds +import pkg/kvstore +import pkg/taskpools import pkg/questionable import pkg/questionable/results import pkg/archivist/marketplace/sales @@ -24,9 +25,7 @@ import ./helpers/periods import ./mockstorage asyncchecksuite "Sales - start": - let - proof = Groth16Proof.example - metaTmp = TempLevelDb.new() + let proof = Groth16Proof.example var request: StorageRequest var sales: Sales @@ -35,6 +34,8 @@ asyncchecksuite "Sales - start": var clock: MockClock var queue: SlotQueue var itemsProcessed: seq[SlotQueueItem] + var metaTp: Taskpool + var metaDs: KVStore setup: request = StorageRequest( @@ -53,7 +54,8 @@ asyncchecksuite "Sales - start": marketplace = MockMarketplace.new() clock = MockClock.new() - let metaDs = TypedDatastore.init(metaTmp.newDb()) + metaTp = Taskpool.new(num_threads = 4) + metaDs = SQLiteKVStore.new(SqliteMemory, metaTp).tryGet() let availability = AvailabilityStore.new(metaDs) storage = MockStorage.new() sales = Sales.new(marketplace, clock, availability, storage) @@ -62,7 +64,8 @@ asyncchecksuite "Sales - start": teardown: await sales.stop() - await metaTmp.destroyDb() + (await metaDs.close()).tryGet() + metaTp.shutdown() proc fillSlot(slotIdx: uint64 = 0.uint64) {.async.} = let address = await marketplace.getSigner() @@ -99,9 +102,7 @@ asyncchecksuite "Sales - start": ) asyncchecksuite "Sales": - let - proof = Groth16Proof.example - metaTmp = TempLevelDb.new() + let proof = Groth16Proof.example var minPricePerBytePerSecond: TokensPerSecond var request: StorageRequest @@ -111,6 +112,8 @@ asyncchecksuite "Sales": var storage: MockStorage var queue: SlotQueue var itemsProcessed: seq[SlotQueueItem] + var metaTp: Taskpool + var metaDs: KVStore setup: minPricePerBytePerSecond = 1'TokensPerSecond @@ -135,7 +138,8 @@ asyncchecksuite "Sales": marketplace.activeSlots[me] = @[] clock = MockClock.new() - let metaDs = TypedDatastore.init(metaTmp.newDb()) + metaTp = Taskpool.new(num_threads = 4) + metaDs = SQLiteKVStore.new(SqliteMemory, metaTp).tryGet() let availability = AvailabilityStore.new(metaDs) sales = Sales.new(marketplace, clock, availability, storage) queue = sales.context.slotQueue @@ -144,7 +148,8 @@ asyncchecksuite "Sales": teardown: await sales.stop() - await metaTmp.destroyDb() + (await metaDs.close()).tryGet() + metaTp.shutdown() proc isInState(idx: int, state: string): bool = proc description(state: State): string = diff --git a/tests/archivist/node/helpers.nim b/tests/archivist/node/helpers.nim index 2c4605f7..7f2338a9 100644 --- a/tests/archivist/node/helpers.nim +++ b/tests/archivist/node/helpers.nim @@ -7,6 +7,8 @@ import pkg/archivist/archivisttypes import pkg/archivist/chunker import pkg/archivist/stores +import pkg/archivist/clock + import ../../asynctest type CountingStore* = ref object of NetworkStore @@ -19,14 +21,17 @@ proc new*( result = CountingStore(engine: engine, localStore: localStore) method getBlock*( - self: CountingStore, address: BlockAddress + self: CountingStore, treeCid: Cid, index: Natural ): Future[?!Block] {.async: (raises: [CancelledError]).} = - self.lookups.mgetOrPut(address.cid, 0).inc - await procCall getBlock(NetworkStore(self), address) + self.lookups.mgetOrPut(treeCid, 0).inc + await procCall getBlock(NetworkStore(self), treeCid, index) proc toTimesDuration*(d: chronos.Duration): times.Duration = initDuration(seconds = d.seconds) +proc toTimesDuration*(d: SecondsSince1970): times.Duration = + initDuration(seconds = d) + proc drain*( stream: LPStream | Result[lpstream.LPStream, ref CatchableError] ): Future[seq[byte]] {.async.} = @@ -52,7 +57,7 @@ proc drain*( proc pipeChunker*(stream: BufferStream, chunker: Chunker) {.async.} = try: - while (let chunk = await chunker.getBytes(); chunk.len > 0): + while (let chunk = (await chunker.getBytes()).tryGet; chunk.len > 0): await stream.pushData(chunk) finally: await stream.pushEof() diff --git a/tests/archivist/node/tempnode.nim b/tests/archivist/node/tempnode.nim index a8b45ae0..aa7ae683 100644 --- a/tests/archivist/node/tempnode.nim +++ b/tests/archivist/node/tempnode.nim @@ -3,15 +3,16 @@ import pkg/questionable/results import pkg/libp2p/builders import pkg/nitro/wallet import pkg/taskpools +import pkg/kvstore import pkg/archivist/discovery import pkg/archivist/stores import pkg/archivist/blockexchange import pkg/archivist/node -import ../../helpers/templeveldb type TemporaryNode* = ref object - tempRepoDb: TempLevelDb - tempMetaDb: TempLevelDb + repoDs: KVStore + metaDs: KVStore + tp: Taskpool localStore: RepoStore p2p: Switch peerStore: PeerCtxStore @@ -24,11 +25,10 @@ type TemporaryNode* = ref object node: ArchivistNodeRef proc initializeLocalStore(temporary: TemporaryNode) = - temporary.tempRepoDb = TempLevelDb.new() - temporary.tempMetaDb = TempLevelDb.new() - let repoDs = temporary.tempRepoDb.newDb() - let metaDs = temporary.tempMetaDb.newDb() - temporary.localStore = RepoStore.new(repoDs, metaDs) + temporary.tp = Taskpool.new(num_threads = 4) + temporary.repoDs = SQLiteKVStore.new(SqliteMemory, temporary.tp).tryGet() + temporary.metaDs = SQLiteKVStore.new(SqliteMemory, temporary.tp).tryGet() + temporary.localStore = RepoStore.new(temporary.repoDs, temporary.metaDs) proc initializeNetwork(temporary: TemporaryNode) = temporary.p2p = newStandardSwitch() @@ -36,7 +36,9 @@ proc initializeNetwork(temporary: TemporaryNode) = temporary.exchangeNetwork = BlockExcnetwork.new(temporary.p2p) let privateKey = temporary.p2p.peerInfo.privateKey let address = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet() - temporary.discoveryNetwork = Discovery.new(privateKey, announceAddrs = @[address]) + let discoveryDs = SQLiteKVStore.new(SqliteMemory, temporary.tp).tryGet() + temporary.discoveryNetwork = + Discovery.new(privateKey, announceAddrs = @[address], store = discoveryDs) proc initializePendingBlocks(temporary: TemporaryNode) = temporary.pendingBlocks = PendingBlocksManager.new() @@ -63,6 +65,7 @@ proc initializeNode(temporary: TemporaryNode) = temporary.node = ArchivistNodeRef.new( temporary.p2p, temporary.networkStore, + temporary.localStore, temporary.exchangeEngine, temporary.discoveryNetwork, Taskpool.new(), @@ -82,8 +85,9 @@ proc create*(_: type TemporaryNode): Future[TemporaryNode] {.async.} = proc destroy*(temporary: TemporaryNode) {.async.} = await temporary.node.stop() - await temporary.tempRepoDb.destroyDb() - await temporary.tempMetaDb.destroyDb() + (await temporary.repoDs.close()).tryGet() + (await temporary.metaDs.close()).tryGet() + temporary.tp.shutdown() func node*(temporary: TemporaryNode): ArchivistNodeRef = temporary.node diff --git a/tests/archivist/node/testnode.nim b/tests/archivist/node/testnode.nim index 35eb4b51..94bc5fd6 100644 --- a/tests/archivist/node/testnode.nim +++ b/tests/archivist/node/testnode.nim @@ -25,6 +25,7 @@ import pkg/archivist/erasure import pkg/archivist/merkletree import pkg/archivist/blocktype as bt import pkg/archivist/rng +import pkg/archivist/utils import pkg/archivist/node {.all.} @@ -37,7 +38,38 @@ import ../slots/helpers import ./helpers import ./tempnode -asyncchecksuite "Test Node - Basic": +proc overlayCount( + repo: RepoStore +): Future[?!int] {.async: (raises: [CancelledError]).} = + let iter = ?await repo.listOverlays() + let cids = ?await utils.collect(iter) + success(cids.len) + +proc assertOverlayCompleted( + repo: RepoStore, treeCid: Cid +): Future[?!void] {.async: (raises: [CancelledError]).} = + let meta = ?await repo.getOverlay(treeCid) + if meta.status != Completed: + return failure("Expected Completed overlay status") + success() + +proc assertRequestOverlaysCompleted( + repo: RepoStore, request: StorageRequest +): Future[?!void] {.async: (raises: [CancelledError]).} = + let + manifestBlk = ?await repo.getBlock(request.content.cid) + manifest = ?Manifest.decode(manifestBlk) + + if not manifest.verifiable: + return failure("Expected verifiable manifest in storage request") + + ?await repo.assertOverlayCompleted(manifest.treeCid) + for slotRoot in manifest.slotRoots: + ?await repo.assertOverlayCompleted(slotRoot) + + success() + +suite "Test Node - Basic": var temporary: TemporaryNode var node: ArchivistNodeRef var localStore: RepoStore @@ -59,7 +91,7 @@ asyncchecksuite "Test Node - Basic": test "Fetch Manifest": let - manifest = await storeDataGetManifest(localStore, chunker) + manifest = (await storeDataGetManifest(localStore, chunker)).tryGet() manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() @@ -72,7 +104,7 @@ asyncchecksuite "Test Node - Basic": fetched == manifest test "Block Batching": - let manifest = await storeDataGetManifest(localStore, chunker) + let manifest = (await storeDataGetManifest(localStore, chunker)).tryGet() for batchSize in 1 .. 12: ( @@ -88,7 +120,8 @@ asyncchecksuite "Test Node - Basic": ).tryGet() test "Block Batching with corrupted blocks": - let blocks = await makeRandomBlocks(datasetSize = 64.KiBs.int, blockSize = 64.KiBs) + let blocks = + (await makeRandomBlocks(datasetSize = 64.KiBs.int, blockSize = 64.KiBs)).tryGet assert blocks.len == 1 let blk = blocks[0] @@ -97,7 +130,7 @@ asyncchecksuite "Test Node - Basic": let pos = rng.Rng.instance.rand(blk.data.len - 1) blk.data[pos] = byte 0 - let manifest = await storeDataGetManifest(localStore, blocks) + let manifest = (await storeDataGetManifest(localStore, blocks)).tryGet() let batchSize = manifest.blocksCount let res = ( @@ -111,7 +144,6 @@ asyncchecksuite "Test Node - Basic": ) ) check res.isFailure - check res.error.msg == "Some blocks failed (Result) to fetch (1)" test "Should store Data Stream": let @@ -123,7 +155,7 @@ asyncchecksuite "Test Node - Basic": var original: seq[byte] try: - while (let chunk = await oddChunker.getBytes(); chunk.len > 0): + while (let chunk = (await oddChunker.getBytes()).tryGet; chunk.len > 0): original &= chunk await stream.pushData(chunk) finally: @@ -147,7 +179,7 @@ asyncchecksuite "Test Node - Basic": test "Should retrieve a Data Stream": let - manifest = await storeDataGetManifest(localStore, chunker) + manifest = (await storeDataGetManifest(localStore, chunker)).tryGet() manifestBlk = bt.Block.new(data = manifest.encode().tryGet, codec = ManifestCodec).tryGet() @@ -187,9 +219,9 @@ asyncchecksuite "Test Node - Basic": test "Should delete an entire dataset": let - blocks = await makeRandomBlocks(datasetSize = 2048, blockSize = 256'nb) - manifest = await storeDataGetManifest(localStore, blocks) - manifestBlock = (await networkStore.storeManifest(manifest)).tryGet() + blocks = (await makeRandomBlocks(datasetSize = 2048, blockSize = 256'nb)).tryGet + manifest = (await storeDataGetManifest(localStore, blocks)).tryGet() + manifestBlock = (await localStore.storeManifest(manifest)).tryGet() manifestCid = manifestBlock.cid check await manifestCid in localStore @@ -202,40 +234,52 @@ asyncchecksuite "Test Node - Basic": for blk in blocks: check not (await blk.cid in localStore) -asyncchecksuite "Test Node - Purchase request": +suite "Test Node - Purchase request": var temporary: TemporaryNode var node: ArchivistNodeRef var localStore: RepoStore + var networkStore: NetworkStore var file: File - var builder: Poseidon2Builder var manifest: Manifest var manifestBlock: bt.Block - var protected: Manifest var protectedManifestBlock: bt.Block - var verifiable: Manifest var verifiableBlock: bt.Block + var verifiableMerkleRoot: array[32, byte] setup: temporary = await TemporaryNode.create() node = temporary.node localStore = temporary.localStore + networkStore = temporary.networkStore file = open(currentSourcePath().parentDir / ".." / ".." / "fixtures" / "test.jpg") let chunker = FileChunker.new(file = file, chunkSize = DefaultBlockSize) - erasure = - Erasure.new(localStore, leoEncoderProvider, leoDecoderProvider, Taskpool.new()) + referenceBlocks = ( + await makeRandomBlocks( + datasetSize = 4 * DefaultBlockSize.int, blockSize = DefaultBlockSize + ) + ).tryGet() - manifest = await storeDataGetManifest(localStore, chunker) + manifest = (await storeDataGetManifest(localStore, chunker)).tryGet() manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - protected = (await erasure.encode(manifest, 3, 2)).tryGet() + + let + referenceManifest = + (await storeDataGetManifest(localStore, referenceBlocks)).tryGet() + erasure = Erasure.new( + networkStore, localStore, leoEncoderProvider, leoDecoderProvider, Taskpool.new() + ) + protected = (await erasure.encode(referenceManifest, 3, 2)).tryGet() + builder = Poseidon2Builder.new(networkStore, localStore, protected).tryGet() + verifiable = (await builder.buildManifest()).tryGet() + protectedManifestBlock = bt.Block.new(protected.encode().tryGet(), codec = ManifestCodec).tryGet() - builder = Poseidon2Builder.new(localStore, protected).tryGet() - verifiable = (await builder.buildManifest()).tryGet() verifiableBlock = bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet() + verifiableMerkleRoot = (verifiable.verifyRoot.fromVerifyCid).tryGet().toBytes teardown: file.close() @@ -255,12 +299,18 @@ asyncchecksuite "Test Node - Purchase request": expiry = 200'StorageDuration, collateralPerByte = 1'Tokens, ) - ).tryGet + ).tryGet() + + let + requestManifestBlock = (await localStore.getBlock(request.content.cid)).tryGet() + requestManifest = Manifest.decode(requestManifestBlock).tryGet() + verifyRoot = (requestManifest.verifyRoot.fromVerifyCid).tryGet().toBytes check: - (await verifiableBlock.cid in localStore) == true - request.content.cid == verifiableBlock.cid - request.content.merkleRoot == builder.verifyRoot.get.toBytes + requestManifest.protected == true + requestManifest.verifiable == true + request.content.merkleRoot == verifyRoot + (await assertRequestOverlaysCompleted(localStore, request)).tryGet() test "Setup purchase request - Protected manifest": (await localStore.putBlock(protectedManifestBlock)).tryGet() @@ -278,10 +328,16 @@ asyncchecksuite "Test Node - Purchase request": ) ).tryGet + let + requestManifestBlock = (await localStore.getBlock(request.content.cid)).tryGet() + requestManifest = Manifest.decode(requestManifestBlock).tryGet() + verifyRoot = (requestManifest.verifyRoot.fromVerifyCid).tryGet().toBytes + check: - (await verifiableBlock.cid in localStore) == true - request.content.cid == verifiableBlock.cid - request.content.merkleRoot == builder.verifyRoot.get.toBytes + requestManifest.protected == true + requestManifest.verifiable == true + request.content.merkleRoot == verifyRoot + (await assertRequestOverlaysCompleted(localStore, request)).tryGet() test "Setup purchase request - Verifiable manifest": (await localStore.putBlock(verifiableBlock)).tryGet() @@ -301,10 +357,12 @@ asyncchecksuite "Test Node - Purchase request": check: request.content.cid == verifiableBlock.cid - request.content.merkleRoot == builder.verifyRoot.get.toBytes + request.content.merkleRoot == verifiableMerkleRoot + (await assertRequestOverlaysCompleted(localStore, request)).tryGet() test "Setup purchase request - Verifiable manifest fails when ec params mismatch": (await localStore.putBlock(verifiableBlock)).tryGet() + let overlaysBefore = (await overlayCount(localStore)).tryGet() let request = ( await node.setupRequest( @@ -323,3 +381,4 @@ asyncchecksuite "Test Node - Purchase request": check request.error.msg == "Attempt to proceed with protected manifest with parameters " & "3/2 but required: 4/2" + check (await overlayCount(localStore)).tryGet() == overlaysBefore diff --git a/tests/archivist/node/testslotrepair.nim b/tests/archivist/node/testslotrepair.nim index 8818c5f8..1a07b789 100644 --- a/tests/archivist/node/testslotrepair.nim +++ b/tests/archivist/node/testslotrepair.nim @@ -22,6 +22,7 @@ import ../../examples import ../helpers import ./helpers +import ../helpers/nodeutils proc fetchStreamData(stream: LPStream, datasetSize: int): Future[seq[byte]] {.async.} = var buf = newSeq[byte](datasetSize) @@ -34,7 +35,7 @@ proc flatten[T](s: seq[seq[T]]): seq[T] = t &= ss return t -asyncchecksuite "Test Node - Slot Repair": +suite "Test Node - Slot Repair": let numNodes = 12 config = NodeConfig( @@ -52,12 +53,13 @@ asyncchecksuite "Test Node - Slot Repair": cluster: NodesCluster nodes: seq[ArchivistNodeRef] - localStores: seq[BlockStore] + localStores: seq[RepoStore] setup: cluster = generateNodes(numNodes, config = config) nodes = cluster.nodes localStores = cluster.localStores + await connectNodes(cluster) teardown: await cluster.cleanup() @@ -66,15 +68,16 @@ asyncchecksuite "Test Node - Slot Repair": test "repair slots (2,1)": let - expiry = (getTime() + DefaultBlockTtl.toTimesDuration + 1.hours).toUnix + expiry = (getTime() + DefaultOverlayTtl.toTimesDuration + 1.hours).toUnix numBlocks = 5 datasetSize = numBlocks * DefaultBlockSize.int ecK = 2 ecM = 1 localStore = localStores[0] store = nodes[0].blockStore - blocks = + blocks = ( await makeRandomBlocks(datasetSize = datasetSize, blockSize = DefaultBlockSize) + ).tryGet data = ( block: collect(newSeq): @@ -84,17 +87,18 @@ asyncchecksuite "Test Node - Slot Repair": check blocks.len == numBlocks # Populate manifest in local store - manifest = await storeDataGetManifest(localStore, blocks) + manifest = (await storeDataGetManifest(localStore, blocks)).tryGet() let manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - erasure = - Erasure.new(store, leoEncoderProvider, leoDecoderProvider, cluster.taskpool) + erasure = Erasure.new( + store, localStore, leoEncoderProvider, leoDecoderProvider, cluster.taskpool + ) (await localStore.putBlock(manifestBlock)).tryGet() protected = (await erasure.encode(manifest, ecK, ecM)).tryGet() - builder = Poseidon2Builder.new(localStore, protected).tryGet() + builder = Poseidon2Builder.new(store, localStore, protected).tryGet() verifiable = (await builder.buildManifest()).tryGet() verifiableBlock = bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet() @@ -149,15 +153,16 @@ asyncchecksuite "Test Node - Slot Repair": test "repair slots (3,2)": let - expiry = (getTime() + DefaultBlockTtl.toTimesDuration + 1.hours).toUnix + expiry = (getTime() + DefaultOverlayTtl.toTimesDuration + 1.hours).toUnix numBlocks = 40 datasetSize = numBlocks * DefaultBlockSize.int ecK = 3 ecM = 2 localStore = localStores[0] store = nodes[0].blockStore - blocks = + blocks = ( await makeRandomBlocks(datasetSize = datasetSize, blockSize = DefaultBlockSize) + ).tryGet data = ( block: collect(newSeq): @@ -167,17 +172,18 @@ asyncchecksuite "Test Node - Slot Repair": check blocks.len == numBlocks # Populate manifest in local store - manifest = await storeDataGetManifest(localStore, blocks) + manifest = (await storeDataGetManifest(localStore, blocks)).tryGet() let manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - erasure = - Erasure.new(store, leoEncoderProvider, leoDecoderProvider, cluster.taskpool) + erasure = Erasure.new( + store, localStore, leoEncoderProvider, leoDecoderProvider, cluster.taskpool + ) (await localStore.putBlock(manifestBlock)).tryGet() protected = (await erasure.encode(manifest, ecK, ecM)).tryGet() - builder = Poseidon2Builder.new(localStore, protected).tryGet() + builder = Poseidon2Builder.new(store, localStore, protected).tryGet() verifiable = (await builder.buildManifest()).tryGet() verifiableBlock = bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet() diff --git a/tests/archivist/slots/backends/testcircomcompat.nim b/tests/archivist/slots/backends/testcircomcompat.nim index d826cdc3..aec486d9 100644 --- a/tests/archivist/slots/backends/testcircomcompat.nim +++ b/tests/archivist/slots/backends/testcircomcompat.nim @@ -5,6 +5,8 @@ import ../../../asynctest import pkg/chronos import pkg/poseidon2 import pkg/serde/json +import pkg/kvstore +import pkg/taskpools import pkg/archivist/slots {.all.} import pkg/archivist/slots/types {.all.} @@ -62,11 +64,8 @@ suite "Test Circom Compat Backend": wasm = "tests/circuits/fixtures/proof_main.wasm" zkey = "tests/circuits/fixtures/proof_main.zkey" - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() - var - store: BlockStore + store: RepoStore manifest: Manifest protected: Manifest verifiable: Manifest @@ -75,11 +74,13 @@ suite "Test Circom Compat Backend": challenge: array[32, byte] builder: Poseidon2Builder sampler: Poseidon2Sampler + tp: Taskpool setup: + tp = Taskpool.new(num_threads = 4) let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() + repoDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() store = RepoStore.new(repoDs, metaDs) @@ -87,7 +88,7 @@ suite "Test Circom Compat Backend": store, numDatasetBlocks, ecK, ecM, blockSize, cellSize ) - builder = Poseidon2Builder.new(store, verifiable).tryGet + builder = Poseidon2Builder.new(store, store, verifiable).tryGet sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet circom = CircomCompatBackendRef.new(r1cs, wasm, zkey).tryGet @@ -97,8 +98,8 @@ suite "Test Circom Compat Backend": teardown: circom.release() # this comes from the rust FFI - await repoTmp.destroyDb() - await metaTmp.destroyDb() + await store.close() + tp.shutdown() test "Should verify with correct input": var proof = (await circom.prove(proofInputs)).tryGet diff --git a/tests/archivist/slots/backends/testnimgroth16.nim b/tests/archivist/slots/backends/testnimgroth16.nim index 5c10459f..b846f8bc 100644 --- a/tests/archivist/slots/backends/testnimgroth16.nim +++ b/tests/archivist/slots/backends/testnimgroth16.nim @@ -7,6 +7,7 @@ import pkg/chronos import pkg/poseidon2 import pkg/serde/json import pkg/taskpools +import pkg/kvstore import pkg/archivist/slots {.all.} import pkg/archivist/slots/types {.all.} @@ -70,11 +71,8 @@ suite "Test NimGoth16 Backend": r1cs = "tests/circuits/fixtures/proof_main.r1cs" zkey = "tests/circuits/fixtures/proof_main.zkey" - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() - var - store: BlockStore + store: RepoStore manifest: Manifest protected: Manifest verifiable: Manifest @@ -83,11 +81,13 @@ suite "Test NimGoth16 Backend": challenge: array[32, byte] builder: Poseidon2Builder sampler: Poseidon2Sampler + tp: Taskpool setup: + tp = Taskpool.new(num_threads = 4) let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() + repoDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() store = RepoStore.new(repoDs, metaDs) @@ -95,7 +95,7 @@ suite "Test NimGoth16 Backend": store, numDatasetBlocks, ecK, ecM, blockSize, cellSize ) - builder = Poseidon2Builder.new(store, verifiable).tryGet + builder = Poseidon2Builder.new(store, store, verifiable).tryGet sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet nimGroth16 = NimGroth16BackendRef.new(graph, r1cs, zkey, tp = Taskpool.new()).tryGet @@ -105,8 +105,8 @@ suite "Test NimGoth16 Backend": teardown: nimGroth16.release() - await repoTmp.destroyDb() - await metaTmp.destroyDb() + await store.close() + tp.shutdown() test "Should verify with correct input": var proof = (await nimGroth16.prove(proofInputs)).tryGet diff --git a/tests/archivist/slots/helpers.nim b/tests/archivist/slots/helpers.nim index be9df940..73daff7b 100644 --- a/tests/archivist/slots/helpers.nim +++ b/tests/archivist/slots/helpers.nim @@ -15,79 +15,14 @@ import pkg/archivist/rng import ../helpers -proc makeManifestBlock*(manifest: Manifest): ?!bt.Block = - without encodedVerifiable =? manifest.encode(), err: - trace "Unable to encode manifest" - return failure(err) - - without blk =? bt.Block.new(data = encodedVerifiable, codec = ManifestCodec), error: - trace "Unable to create block from manifest" - return failure(error) - - success blk - -proc storeManifest*( - store: BlockStore, manifest: Manifest -): Future[?!bt.Block] {.async.} = - without blk =? makeManifestBlock(manifest), err: - trace "Unable to create manifest block", err = err.msg - return failure(err) - - if err =? (await store.putBlock(blk)).errorOption: - trace "Unable to store manifest block", cid = blk.cid, err = err.msg - return failure(err) - - success blk - -proc makeManifest*( - cids: seq[Cid], - datasetSize: NBytes, - blockSize: NBytes, - store: BlockStore, - hcodec = Sha256HashCodec, - dataCodec = BlockCodec, -): Future[?!Manifest] {.async.} = - without tree =? ArchivistTree.init(cids), err: - return failure(err) - - without treeCid =? tree.rootCid(CIDv1, dataCodec), err: - return failure(err) - - for index, cid in cids: - without proof =? tree.getProof(index), err: - return failure(err) - - if err =? (await store.putCidAndProof(treeCid, index, cid, proof)).errorOption: - # TODO add log here - return failure(err) - - let manifest = Manifest.new( - treeCid = treeCid, - blockSize = blockSize, - datasetSize = datasetSize, - version = CIDv1, - hcodec = hcodec, - codec = dataCodec, - ) - - without manifestBlk =? await store.storeManifest(manifest), err: - trace "Unable to store manifest" - return failure(err) - - success manifest - -proc createBlocks*( - chunker: Chunker, store: BlockStore -): Future[seq[bt.Block]] {.async.} = +proc createBlocks*(chunker: Chunker): Future[seq[bt.Block]] {.async.} = collect(newSeq): - while (let chunk = await chunker.getBytes(); chunk.len > 0): - let blk = bt.Block.new(chunk).tryGet() - discard await store.putBlock(blk) - blk + while (let chunk = (await chunker.getBytes()).tryGet; chunk.len > 0): + bt.Block.new(chunk).tryGet() proc createProtectedManifest*( datasetBlocks: seq[bt.Block], - store: BlockStore, + store: RepoStore, numDatasetBlocks: int, ecK: int, ecM: int, @@ -99,17 +34,26 @@ proc createProtectedManifest*( cids = datasetBlocks.mapIt(it.cid) datasetTree = ArchivistTree.init(cids[0 ..< numDatasetBlocks]).tryGet() datasetTreeCid = datasetTree.rootCid().tryGet() - protectedTree = ArchivistTree.init(cids).tryGet() protectedTreeCid = protectedTree.rootCid().tryGet() - for index, cid in cids[0 ..< numDatasetBlocks]: + # Create overlay for dataset tree and store blocks + proofs + let tmpDatasetCid = (await store.createTmpOverlay()).tryGet() + for i, blk in datasetBlocks[0 ..< numDatasetBlocks]: + (await store.putBlock(tmpDatasetCid, blk, i)).tryGet() + (await store.finalizeOverlay(tmpDatasetCid, datasetTreeCid)).tryGet() + for index in 0 ..< numDatasetBlocks: let proof = datasetTree.getProof(index).tryGet() - (await store.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet + (await store.putCidAndProof(datasetTreeCid, index, cids[index], proof)).tryGet() + # Create overlay for protected tree and store all blocks + proofs + let tmpProtectedCid = (await store.createTmpOverlay()).tryGet() + for i, blk in datasetBlocks: + (await store.putBlock(tmpProtectedCid, blk, i)).tryGet() + (await store.finalizeOverlay(tmpProtectedCid, protectedTreeCid)).tryGet() for index, cid in cids: let proof = protectedTree.getProof(index).tryGet() - (await store.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet + (await store.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet() let manifest = Manifest.new( @@ -127,19 +71,13 @@ proc createProtectedManifest*( strategy = SteppedStrategy, ) - manifestBlock = - bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - - protectedManifestBlock = - bt.Block.new(protectedManifest.encode().tryGet(), codec = ManifestCodec).tryGet() - - (await store.putBlock(manifestBlock)).tryGet() - (await store.putBlock(protectedManifestBlock)).tryGet() + discard (await store.storeManifest(manifest)).tryGet() + discard (await store.storeManifest(protectedManifest)).tryGet() (manifest, protectedManifest) proc createVerifiableManifest*( - store: BlockStore, + store: RepoStore, numDatasetBlocks: int, ecK: int, ecM: int, @@ -151,22 +89,20 @@ proc createVerifiableManifest*( let numSlots = ecK + ecM numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) - # total number of blocks in the dataset after - # EC (should will match number of slots) originalDatasetSize = numDatasetBlocks * blockSize.int totalDatasetSize = numTotalBlocks * blockSize.int chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize) - datasetBlocks = await chunker.createBlocks(store) + datasetBlocks = await chunker.createBlocks() (manifest, protectedManifest) = await createProtectedManifest( datasetBlocks, store, numDatasetBlocks, ecK, ecM, blockSize, originalDatasetSize, totalDatasetSize, ) - builder = Poseidon2Builder.new(store, protectedManifest, cellSize = cellSize).tryGet + builder = + Poseidon2Builder.new(store, store, protectedManifest, cellSize = cellSize).tryGet verifiableManifest = (await builder.buildManifest()).tryGet - # build the slots and manifest (manifest, protectedManifest, verifiableManifest) diff --git a/tests/archivist/slots/sampler/testsampler.nim b/tests/archivist/slots/sampler/testsampler.nim index f83e22cd..a34d5232 100644 --- a/tests/archivist/slots/sampler/testsampler.nim +++ b/tests/archivist/slots/sampler/testsampler.nim @@ -4,6 +4,8 @@ import std/options import ../../../asynctest import pkg/questionable/results +import pkg/kvstore +import pkg/taskpools import pkg/archivist/stores import pkg/archivist/merkletree @@ -78,8 +80,6 @@ suite "Test Sampler": entropy = 1234567.toF blockSize = DefaultBlockSize cellSize = DefaultCellSize - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() var store: RepoStore @@ -87,11 +87,13 @@ suite "Test Sampler": manifest: Manifest protected: Manifest verifiable: Manifest + tp: Taskpool setup: + tp = Taskpool.new(num_threads = 4) let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() + repoDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() store = RepoStore.new(repoDs, metaDs) @@ -100,12 +102,11 @@ suite "Test Sampler": ) # create sampler - builder = Poseidon2Builder.new(store, verifiable).tryGet + builder = Poseidon2Builder.new(store, store, verifiable).tryGet teardown: await store.close() - await repoTmp.destroyDb() - await metaTmp.destroyDb() + tp.shutdown() test "Should fail instantiating for invalid slot index": let sampler = Poseidon2Sampler.new(builder.slotRoots.len, store, builder) @@ -114,7 +115,7 @@ suite "Test Sampler": test "Should fail instantiating for non verifiable builder": let - nonVerifiableBuilder = Poseidon2Builder.new(store, protected).tryGet + nonVerifiableBuilder = Poseidon2Builder.new(store, store, protected).tryGet sampler = Poseidon2Sampler.new(slotIndex, store, nonVerifiableBuilder) check sampler.isErr diff --git a/tests/archivist/slots/sampler/testutils.nim b/tests/archivist/slots/sampler/testutils.nim index b84b528d..a6b13dca 100644 --- a/tests/archivist/slots/sampler/testutils.nim +++ b/tests/archivist/slots/sampler/testutils.nim @@ -14,7 +14,6 @@ import pkg/archivist/blocktype as bt import pkg/archivist/marketplace/contracts/requests import pkg/archivist/marketplace/contracts import pkg/archivist/merkletree -import pkg/archivist/stores/cachestore import pkg/archivist/slots/types import pkg/archivist/slots/sampler/utils import pkg/archivist/utils/json diff --git a/tests/archivist/slots/testprover.nim b/tests/archivist/slots/testprover.nim index 21ad21ab..f860e512 100644 --- a/tests/archivist/slots/testprover.nim +++ b/tests/archivist/slots/testprover.nim @@ -15,6 +15,7 @@ import pkg/archivist/utils/poseidon2digest import pkg/archivist/nat import pkg/archivist/utils/natutils import pkg/taskpools +import pkg/kvstore import ./helpers import ../helpers @@ -23,19 +24,19 @@ suite "Test CircomCompat Prover": samples = 5 blockSize = DefaultBlockSize cellSize = DefaultCellSize - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() tp = Taskpool.new() challenge = 1234567.toF.toBytes.toArray32 var - store: BlockStore + store: RepoStore prover: Prover + dbTp: Taskpool setup: + dbTp = Taskpool.new(num_threads = 4) let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() + repoDs = SQLiteKVStore.new(SqliteMemory, dbTp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, dbTp).tryGet() backend = CircomCompatBackendRef.new( r1csPath = "tests/circuits/fixtures/proof_main.r1cs", wasmPath = "tests/circuits/fixtures/proof_main.wasm", @@ -47,8 +48,8 @@ suite "Test CircomCompat Prover": prover = Prover.new(backend, samples, tp) teardown: - await repoTmp.destroyDb() - await metaTmp.destroyDb() + await store.close() + dbTp.shutdown() test "Should sample and prove a slot": let @@ -61,8 +62,9 @@ suite "Test CircomCompat Prover": cellSize, ) - builder = - Poseidon2Builder.new(store, verifiable, verifiable.verifiableStrategy).tryGet + builder = Poseidon2Builder.new( + store, store, verifiable, verifiable.verifiableStrategy + ).tryGet sampler = Poseidon2Sampler.new(1, store, builder).tryGet (_, checked) = (await prover.prove(sampler, verifiable, challenge, verify = true)).tryGet @@ -85,8 +87,9 @@ suite "Test CircomCompat Prover": cellSize, ) - builder = - Poseidon2Builder.new(store, verifiable, verifiable.verifiableStrategy).tryGet + builder = Poseidon2Builder.new( + store, store, verifiable, verifiable.verifiableStrategy + ).tryGet sampler = Poseidon2Sampler.new(1, store, builder).tryGet (_, checked) = (await prover.prove(sampler, verifiable, challenge, verify = true)).tryGet @@ -99,20 +102,20 @@ suite "Test NimGroth16 Prover": samples = 5 blockSize = DefaultBlockSize cellSize = DefaultCellSize - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() tp = Taskpool.new() challenge = 1234567.toF.toBytes.toArray32 var - store: BlockStore + store: RepoStore prover: Prover + dbTp: Taskpool setup: + dbTp = Taskpool.new(num_threads = 4) let tp = Taskpool.new() - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() + repoDs = SQLiteKVStore.new(SqliteMemory, dbTp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, dbTp).tryGet() backend = NimGroth16BackendRef.new( r1csPath = "tests/circuits/fixtures/proof_main.r1cs", graphPath = "tests/circuits/fixtures/proof_main.bin", @@ -124,8 +127,8 @@ suite "Test NimGroth16 Prover": prover = Prover.new(backend, samples, tp) teardown: - await repoTmp.destroyDb() - await metaTmp.destroyDb() + await store.close() + dbTp.shutdown() test "Should sample and prove a slot": let @@ -138,8 +141,9 @@ suite "Test NimGroth16 Prover": cellSize, ) - builder = - Poseidon2Builder.new(store, verifiable, verifiable.verifiableStrategy).tryGet + builder = Poseidon2Builder.new( + store, store, verifiable, verifiable.verifiableStrategy + ).tryGet sampler = Poseidon2Sampler.new(1, store, builder).tryGet (_, checked) = (await prover.prove(sampler, verifiable, challenge, verify = true)).tryGet @@ -162,8 +166,9 @@ suite "Test NimGroth16 Prover": cellSize, ) - builder = - Poseidon2Builder.new(store, verifiable, verifiable.verifiableStrategy).tryGet + builder = Poseidon2Builder.new( + store, store, verifiable, verifiable.verifiableStrategy + ).tryGet sampler = Poseidon2Sampler.new(1, store, builder).tryGet (_, checked) = (await prover.prove(sampler, verifiable, challenge, verify = true)).tryGet diff --git a/tests/archivist/slots/testslotbuilder.nim b/tests/archivist/slots/testslotbuilder.nim index 70884aaf..4607b655 100644 --- a/tests/archivist/slots/testslotbuilder.nim +++ b/tests/archivist/slots/testslotbuilder.nim @@ -5,6 +5,8 @@ import ../../asynctest import pkg/chronos import pkg/questionable/results +import pkg/kvstore +import pkg/taskpools import pkg/archivist/blocktype as bt import pkg/archivist/rng import pkg/archivist/stores @@ -23,6 +25,7 @@ import ../merkletree/helpers import pkg/archivist/indexingstrategy {.all.} import pkg/archivist/slots {.all.} +import pkg/archivist/stores/repostore/operations privateAccess(Poseidon2Builder) # enable access to private fields privateAccess(Manifest) # enable access to private fields @@ -62,26 +65,26 @@ suite "Slot builder": # empty digest emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int) - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() var datasetBlocks: seq[bt.Block] - localStore: BlockStore + localStore: RepoStore manifest: Manifest protectedManifest: Manifest builder: Poseidon2Builder chunker: Chunker + tp: Taskpool setup: + tp = Taskpool.new(num_threads = 4) let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() + repoDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() localStore = RepoStore.new(repoDs, metaDs) chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize) - datasetBlocks = await chunker.createBlocks(localStore) + datasetBlocks = await chunker.createBlocks() (manifest, protectedManifest) = await createProtectedManifest( datasetBlocks, localStore, numDatasetBlocks, ecK, ecM, blockSize, @@ -90,8 +93,7 @@ suite "Slot builder": teardown: await localStore.close() - await repoTmp.destroyDb() - await metaTmp.destroyDb() + tp.shutdown() # TODO: THIS IS A BUG IN asynctest, because it doesn't release the # objects after the test is done, so we need to do it manually @@ -113,8 +115,9 @@ suite "Slot builder": ) check: - Poseidon2Builder.new(localStore, unprotectedManifest, cellSize = cellSize).error.msg == - "Manifest is not protected." + Poseidon2Builder.new( + localStore, localStore, unprotectedManifest, cellSize = cellSize + ).error.msg == "Manifest is not protected." test "Number of blocks must be devisable by number of slots": let mismatchManifest = Manifest.new( @@ -131,8 +134,9 @@ suite "Slot builder": ) check: - Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize).error.msg == - "Number of blocks must be divisible by number of slots." + Poseidon2Builder.new( + localStore, localStore, mismatchManifest, cellSize = cellSize + ).error.msg == "Number of blocks must be divisible by number of slots." test "Block size must be divisable by cell size": let mismatchManifest = Manifest.new( @@ -149,12 +153,14 @@ suite "Slot builder": ) check: - Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize).error.msg == - "Block size must be divisible by cell size." + Poseidon2Builder.new( + localStore, localStore, mismatchManifest, cellSize = cellSize + ).error.msg == "Block size must be divisible by cell size." test "Should build correct slot builder": - builder = - Poseidon2Builder.new(localStore, protectedManifest, cellSize = cellSize).tryGet() + builder = Poseidon2Builder + .new(localStore, localStore, protectedManifest, cellSize = cellSize) + .tryGet() check: builder.cellSize == cellSize @@ -171,7 +177,7 @@ suite "Slot builder": ) builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) + .new(localStore, localStore, protectedManifest, cellSize = cellSize) .tryGet() for i in 0 ..< numSlots: @@ -196,7 +202,7 @@ suite "Slot builder": ) builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) + .new(localStore, localStore, protectedManifest, cellSize = cellSize) .tryGet() for i in 0 ..< numSlots: @@ -215,8 +221,9 @@ suite "Slot builder": slotTree.root().tryGet() == expectedRoot test "Should persist trees for all slots": - let builder = - Poseidon2Builder.new(localStore, protectedManifest, cellSize = cellSize).tryGet() + let builder = Poseidon2Builder + .new(localStore, localStore, protectedManifest, cellSize = cellSize) + .tryGet() for i in 0 ..< numSlots: let @@ -242,7 +249,7 @@ suite "Slot builder": 0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks ) builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) + .new(localStore, localStore, protectedManifest, cellSize = cellSize) .tryGet() (await builder.buildSlots()).tryGet @@ -270,7 +277,7 @@ suite "Slot builder": 0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks ) builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) + .new(localStore, localStore, protectedManifest, cellSize = cellSize) .tryGet() slotsHashes = collect(newSeq): @@ -296,46 +303,182 @@ suite "Slot builder": test "Should not build from verifiable manifest with 0 slots": var builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) + .new(localStore, localStore, protectedManifest, cellSize = cellSize) .tryGet() verifyManifest = (await builder.buildManifest()).tryGet() verifyManifest.slotRoots = @[] - check Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).isErr + check Poseidon2Builder.new( + localStore, localStore, verifyManifest, cellSize = cellSize + ).isErr test "Should not build from verifiable manifest with incorrect number of slots": var builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) + .new(localStore, localStore, protectedManifest, cellSize = cellSize) .tryGet() verifyManifest = (await builder.buildManifest()).tryGet() verifyManifest.slotRoots.del(verifyManifest.slotRoots.len - 1) - check Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).isErr + check Poseidon2Builder.new( + localStore, localStore, verifyManifest, cellSize = cellSize + ).isErr test "Should not build from verifiable manifest with invalid verify root": - let builder = - Poseidon2Builder.new(localStore, protectedManifest, cellSize = cellSize).tryGet() + let builder = Poseidon2Builder + .new(localStore, localStore, protectedManifest, cellSize = cellSize) + .tryGet() var verifyManifest = (await builder.buildManifest()).tryGet() rng.shuffle(Rng.instance, verifyManifest.verifyRoot.data.buffer) - check Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).isErr + check Poseidon2Builder.new( + localStore, localStore, verifyManifest, cellSize = cellSize + ).isErr test "Should build from verifiable manifest": let builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) + .new(localStore, localStore, protectedManifest, cellSize = cellSize) .tryGet() verifyManifest = (await builder.buildManifest()).tryGet() - verificationBuilder = - Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).tryGet() + verificationBuilder = Poseidon2Builder + .new(localStore, localStore, verifyManifest, cellSize = cellSize) + .tryGet() check: builder.slotRoots == verificationBuilder.slotRoots builder.verifyRoot == verificationBuilder.verifyRoot + +suite "Cell-aware slot building": + ## Tests for slot building that verify cell leaves are stored correctly + ## with separate cellCid and blkCid for proper refcount handling. + ## + let + blockSize = NBytes 1024 + cellSize = NBytes 64 + ecK = 2 + ecM = 1 + + numSlots = ecK + ecM + numDatasetBlocks = 3 + numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) + originalDatasetSize = numDatasetBlocks * blockSize.int + totalDatasetSize = numTotalBlocks * blockSize.int + + numSlotBlocks = numTotalBlocks div numSlots + numBlockCells = (blockSize div cellSize).int + numSlotCells = numSlotBlocks * numBlockCells + pow2SlotCells = nextPowerOfTwo(numSlotCells) + numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks + + emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int) + + var + datasetBlocks: seq[bt.Block] + localStore: RepoStore + manifest: Manifest + protectedManifest: Manifest + tp: Taskpool + + setup: + tp = Taskpool.new(num_threads = 4) + let + repoDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + + localStore = RepoStore.new(repoDs, metaDs) + let chunker = + RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize) + datasetBlocks = await chunker.createBlocks() + + (manifest, protectedManifest) = await createProtectedManifest( + datasetBlocks, localStore, numDatasetBlocks, ecK, ecM, blockSize, + originalDatasetSize, totalDatasetSize, + ) + + teardown: + await localStore.close() + tp.shutdown() + reset(datasetBlocks) + reset(localStore) + reset(manifest) + reset(protectedManifest) + + test "buildSlot stores cell leaves with correct cellCid and blkCid": + let builder = Poseidon2Builder + .new(localStore, localStore, protectedManifest, cellSize = cellSize) + .tryGet() + + let slotRoot = (await builder.buildSlot(0)).tryGet() + let slotCid = slotRoot.toSlotCid().tryGet() + + # Verify that leaf metadata has cell flag set + for i in 0 ..< protectedManifest.numSlotBlocks: + let leaf = (await localStore.getLeafMetadata(slotCid, i.Natural)).tryGet() + check: + leaf.isCell == true + leaf.blkCid != Cid() # blkCid should point to a real block + + test "buildSlot stores proofs for pad blocks with empty blkCid": + let builder = Poseidon2Builder + .new(localStore, localStore, protectedManifest, cellSize = cellSize) + .tryGet() + + let slotRoot = (await builder.buildSlot(0)).tryGet() + let slotCid = slotRoot.toSlotCid().tryGet() + + # ALL positions (including pad blocks) should have leaf metadata + let numRealBlocks = protectedManifest.numSlotBlocks + let numTotalSlotBlocks = builder.numSlotBlocks + + # Check that all positions have metadata + for i in 0 ..< numTotalSlotBlocks: + let leafRes = await localStore.getLeafMetadata(slotCid, i.Natural) + check leafRes.isOk + + # If there are pad blocks, their blkCid should be empty CID + if numTotalSlotBlocks > numRealBlocks: + for i in numRealBlocks ..< numTotalSlotBlocks: + let leaf = (await localStore.getLeafMetadata(slotCid, i.Natural)).tryGet() + check leaf.blkCid.isEmpty + + test "buildSlot increments refcount on blkCid, not cellCid": + let builder = Poseidon2Builder + .new(localStore, localStore, protectedManifest, cellSize = cellSize) + .tryGet() + + # Build one slot + let slotRoot = (await builder.buildSlot(0)).tryGet() + let slotCid = slotRoot.toSlotCid().tryGet() + + # Get the first block's CID and check its refcount + let leaf = (await localStore.getLeafMetadata(slotCid, 0.Natural)).tryGet() + let blkRefCount = (await localStore.blockRefCount(leaf.blkCid)).tryGet() + + # Refcount should be at least 1 (this slot references it) + check blkRefCount >= 1 + + # Cell CID should not have a block metadata entry + let cellRefCountRes = await localStore.blockRefCount(leaf.cellCid) + check cellRefCountRes.isErr or cellRefCountRes.tryGet() == 0 + + test "Multiple slots referencing same block increment refcount correctly": + let builder = Poseidon2Builder + .new(localStore, localStore, protectedManifest, cellSize = cellSize) + .tryGet() + + # Build all slots + (await builder.buildSlots()).tryGet() + + # Get refcount on first dataset block + let firstBlockCid = datasetBlocks[0].cid + let refCount = (await localStore.blockRefCount(firstBlockCid)).tryGet() + + # Refcount should be 1 (block is referenced by one slot in EC distribution) + check refCount >= 1 diff --git a/tests/archivist/stores/commonstoretests.nim b/tests/archivist/stores/commonstoretests.nim deleted file mode 100644 index 2c47bb81..00000000 --- a/tests/archivist/stores/commonstoretests.nim +++ /dev/null @@ -1,166 +0,0 @@ -import std/sequtils -import std/strutils -import std/options - -import pkg/chronos -import pkg/libp2p/multicodec -import pkg/stew/byteutils -import pkg/questionable -import pkg/questionable/results -import pkg/archivist/stores/cachestore -import pkg/archivist/chunker -import pkg/archivist/manifest -import pkg/archivist/merkletree -import pkg/archivist/utils - -import ../../asynctest -import ../helpers -import ../examples - -type - StoreProvider* = proc(): BlockStore {.gcsafe.} - Before* = proc(): Future[void] {.gcsafe.} - After* = proc(): Future[void] {.gcsafe.} - -proc commonBlockStoreTests*( - name: string, provider: StoreProvider, before: Before = nil, after: After = nil -) = - asyncchecksuite name & " Store Common": - var - newBlock, newBlock1, newBlock2, newBlock3: Block - manifest: Manifest - tree: ArchivistTree - store: BlockStore - - setup: - newBlock = Block.new("New Kids on the Block".toBytes()).tryGet() - newBlock1 = Block.new("1".repeat(100).toBytes()).tryGet() - newBlock2 = Block.new("2".repeat(100).toBytes()).tryGet() - newBlock3 = Block.new("3".repeat(100).toBytes()).tryGet() - - (manifest, tree) = - makeManifestAndTree(@[newBlock, newBlock1, newBlock2, newBlock3]).tryGet() - - if not isNil(before): - await before() - - store = provider() - - teardown: - await store.close() - - if not isNil(after): - await after() - - test "putBlock": - (await store.putBlock(newBlock1)).tryGet() - check (await store.hasBlock(newBlock1.cid)).tryGet() - - test "putBlock raises onBlockStored": - var storedCid = Cid.example - proc onStored(cid: Cid) {.async: (raises: []).} = - storedCid = cid - - store.onBlockStored = onStored.some() - - (await store.putBlock(newBlock1)).tryGet() - - check storedCid == newBlock1.cid - - test "getBlock": - (await store.putBlock(newBlock)).tryGet() - let blk = await store.getBlock(newBlock.cid) - check blk.tryGet() == newBlock - - test "fail getBlock": - expect BlockNotFoundError: - discard (await store.getBlock(newBlock.cid)).tryGet() - - test "hasBlock": - (await store.putBlock(newBlock)).tryGet() - - check: - (await store.hasBlock(newBlock.cid)).tryGet() - await newBlock.cid in store - - test "fail hasBlock": - check: - not (await store.hasBlock(newBlock.cid)).tryGet() - not (await newBlock.cid in store) - - test "delBlock": - (await store.putBlock(newBlock1)).tryGet() - check (await store.hasBlock(newBlock1.cid)).tryGet() - - (await store.delBlock(newBlock1.cid)).tryGet() - - check not (await store.hasBlock(newBlock1.cid)).tryGet() - - test "listBlocks Blocks": - let - blocks = @[newBlock1, newBlock2, newBlock3] - - putHandles = await allFinished(blocks.mapIt(store.putBlock(it))) - - for handle in putHandles: - check not handle.failed - check handle.read.isOk - - let cidsIter = (await store.listBlocks(blockType = BlockType.Block)).tryGet() - - var count = 0 - for c in cidsIter: - if cid =? await c: - check (await store.hasBlock(cid)).tryGet() - count.inc - - check count == 3 - - test "listBlocks Manifest": - let - blocks = @[newBlock1, newBlock2, newBlock3] - manifestBlock = - Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - treeBlock = Block.new(tree.encode()).tryGet() - putHandles = await allFinished( - (@[treeBlock, manifestBlock] & blocks).mapIt(store.putBlock(it)) - ) - - for handle in putHandles: - check not handle.failed - check handle.read.isOk - - let cidsIter = (await store.listBlocks(blockType = BlockType.Manifest)).tryGet() - - var count = 0 - for c in cidsIter: - if cid =? await c: - check manifestBlock.cid == cid - check (await store.hasBlock(cid)).tryGet() - count.inc - - check count == 1 - - test "listBlocks Both": - let - blocks = @[newBlock1, newBlock2, newBlock3] - manifestBlock = - Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - treeBlock = Block.new(tree.encode()).tryGet() - putHandles = await allFinished( - (@[treeBlock, manifestBlock] & blocks).mapIt(store.putBlock(it)) - ) - - for handle in putHandles: - check not handle.failed - check handle.read.isOk - - let cidsIter = (await store.listBlocks(blockType = BlockType.Both)).tryGet() - - var count = 0 - for c in cidsIter: - if cid =? await c: - check (await store.hasBlock(cid)).tryGet() - count.inc - - check count == 5 diff --git a/tests/archivist/stores/repostore/overlays/helpers.nim b/tests/archivist/stores/repostore/overlays/helpers.nim new file mode 100644 index 00000000..f2656515 --- /dev/null +++ b/tests/archivist/stores/repostore/overlays/helpers.nim @@ -0,0 +1,82 @@ +import pkg/chronos +import pkg/kvstore +import pkg/stew/bitseqs +import pkg/archivist/stores +import pkg/archivist/merkletree +import pkg/archivist/blocktype as bt + +import ../../../helpers + +type + KVStoreProvider* = proc(): KVStore {.gcsafe.} + Before* = proc(): Future[void] {.gcsafe.} + After* = proc(): Future[void] {.gcsafe.} + + BlockBitState* = enum + ## Reasons for BitSeq inconsistency between overlay and leaf metadata + BitSetButLeafDeleted ## Bit is set in BitSeq but leaf is marked deleted + LeafExistsButBitNotSet ## Leaf exists and not deleted but bit not set in BitSeq + BitSetButNoLeafMetadata ## Bit is set in BitSeq but no leaf metadata exists + InvalidKeyFormat ## Key format is invalid + +proc createTestBlock*(size: int): bt.Block = + bt.Block.new('a'.repeat(size).toBytes).tryGet() + +proc putBlockWithOverlay*( + repo: RepoStore, blk: bt.Block +): Future[?!(Cid, Natural)] {.async.} = + let (_, tree) = makeManifestAndTree(@[blk]).tryGet() + let treeCid = tree.rootCid.tryGet() + let proof = tree.getProof(0).tryGet() + var blocks = BitSeq.init(1) + blocks.setBit(0) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = blocks)).tryGet() + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() + success((treeCid, 0.Natural)) + +proc verifyBlockBitState*( + self: RepoStore, treeCid: Cid +): Future[?!seq[(Natural, BlockBitState)]] {.async: (raises: [CancelledError]).} = + ## Verify that the overlay BitSeq is consistent with leaf metadata. + ## + ## Returns a sequence of (index, reason) for each inconsistency found. + ## An empty sequence means the overlay is consistent. + ## + ## Consistency rules: + ## - If bit i is set in BitSeq, leaf metadata must exist at index i and not be deleted + ## - If leaf metadata exists at index i and is not deleted, bit i must be set in BitSeq + ## + ## Note: This is an expensive operation that queries all leaf metadata. + ## Use for debugging and testing only. + ## + + var states: seq[(Natural, BlockBitState)] + let + overlayMeta = ?await self.metaDs.get(?overlayKey(treeCid), OverlayMetadata) + bits = overlayMeta.val.blocks + iter = + ?(await query(self.metaDs, Query.init(?blockLeafQueryKey(treeCid)), LeafMetadata)) + + var leafIndices: HashSet[Natural] + for recordFut in iter: + if record =? ?catch(?(await recordFut)): + let indexStr = record.key.value + without idx =? parseInt(indexStr).catch, err: + states.add((0.Natural, InvalidKeyFormat)) + trace "Invalid leaf metadata key format", key = record.key + continue + leafIndices.incl(idx.Natural) + + if record.val.deleted: + if idx < bits.len and bits[idx]: + states.add((idx.Natural, BitSetButLeafDeleted)) + else: + if idx >= bits.len or not bits[idx]: + states.add((idx.Natural, LeafExistsButBitNotSet)) + + for i in 0 ..< bits.len: + if bits[i] and i notin leafIndices: + states.add((i.Natural, BitSetButNoLeafMetadata)) + + success(states) diff --git a/tests/archivist/stores/repostore/overlays/testcells.nim b/tests/archivist/stores/repostore/overlays/testcells.nim new file mode 100644 index 00000000..a9156751 --- /dev/null +++ b/tests/archivist/stores/repostore/overlays/testcells.nim @@ -0,0 +1,287 @@ +import std/os +import std/tempfiles + +import pkg/questionable +import pkg/questionable/results + +import pkg/chronos +import pkg/taskpools +import pkg/stew/bitseqs +import pkg/kvstore +import pkg/kvstore/fsds +import pkg/libp2p/multicodec + +import pkg/archivist/stores +import pkg/archivist/stores/repostore/operations +import pkg/archivist/stores/repostore/types +import pkg/archivist/blocktype as bt +import pkg/archivist/clock + +import pkg/archivist/merkletree/archivist + +import ../../../../asynctest +import ../../../helpers +import ../../../helpers/mockclock +import ../../../examples + +import ./helpers + +proc testCells*( + name: string, + repoDsProvider: KVStoreProvider, + metaDsProvider: KVStoreProvider, + before: Before = nil, + after: After = nil, +) = + suite name: + var + repoDs: KVStore + metaDs: KVStore + mockClock: MockClock + repo: RepoStore + + let now: SecondsSince1970 = 123 + + setup: + if not isNil(before): + await before() + repoDs = repoDsProvider() + metaDs = metaDsProvider() + mockClock = MockClock.new() + mockClock.set(now) + repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = + 10000'nb) + + teardown: + (await repoDs.close()).tryGet + (await metaDs.close()).tryGet + if not isNil(after): + await after() + + test "Cell leaf stores metadata and refcount behavior": + let + blk1 = createTestBlock(400) + blk2 = createTestBlock(401) + cellCid1 = bt.Block.new("cell1".toBytes).tryGet().cid + cellCid2 = bt.Block.new("cell2".toBytes).tryGet().cid + (_, tree) = makeManifestAndTree(@[blk1, blk2]).tryGet() + treeCid = tree.rootCid.tryGet() + proof1 = tree.getProof(0).tryGet() + proof2 = tree.getProof(1).tryGet() + + (await repo.putOverlay(treeCid, status = Completed.some)).tryGet() + + ( + await repo.putCellCidsAndProofs( + treeCid, + @[ + (0.Natural, cellCid1, blk1.cid, proof1), + (1.Natural, cellCid2, blk2.cid, proof2), + ], + ) + ).tryGet() + + let + leaf1 = (await repo.getLeafMetadata(treeCid, 0.Natural)).tryGet() + leaf2 = (await repo.getLeafMetadata(treeCid, 1.Natural)).tryGet() + blkRefCount1 = (await repo.blockRefCount(blk1.cid)).tryGet() + blkRefCount2 = (await repo.blockRefCount(blk2.cid)).tryGet() + cellRefCountRes1 = await repo.blockRefCount(cellCid1) + cellRefCountRes2 = await repo.blockRefCount(cellCid2) + + check: + leaf1.isCell == true + leaf1.cellCid == cellCid1 + leaf1.blkCid == blk1.cid + leaf2.isCell == true + leaf2.cellCid == cellCid2 + leaf2.blkCid == blk2.cid + blkRefCount1 == 1 + blkRefCount2 == 1 + cellRefCountRes1.isErr + cellRefCountRes2.isErr + + test "Cell leaf refcount increments and decrements correctly": + let + blk = createTestBlock(402) + blk2 = createTestBlock(403) + cellCid1 = bt.Block.new("cell1".toBytes).tryGet().cid + cellCid2 = bt.Block.new("cell2".toBytes).tryGet().cid + (_, tree) = makeManifestAndTree(@[blk, blk2]).tryGet() + treeCid = tree.rootCid.tryGet() + proof1 = tree.getProof(0).tryGet() + proof2 = tree.getProof(1).tryGet() + + (await repo.putOverlay(treeCid, status = Completed.some)).tryGet() + + ( + await repo.putCellCidsAndProofs( + treeCid, + @[ + (0.Natural, cellCid1, blk.cid, proof1), + (1.Natural, cellCid2, blk.cid, proof2), + ], + ) + ).tryGet() + + let refCountBefore = (await repo.blockRefCount(blk.cid)).tryGet() + check refCountBefore == 2 + + (await repo.delLeafBlockMetadata(treeCid, @[0.Natural])).tryGet() + + let refCountAfter = (await repo.blockRefCount(blk.cid)).tryGet() + check refCountAfter == 1 + + test "Cell and regular leaves coexist": + let + blk1 = createTestBlock(410) + blk2 = createTestBlock(411) + cellCid = bt.Block.new("cell".toBytes).tryGet().cid + (_, tree) = makeManifestAndTree(@[blk1, blk2]).tryGet() + treeCid = tree.rootCid.tryGet() + proof1 = tree.getProof(0).tryGet() + proof2 = tree.getProof(1).tryGet() + + (await repo.putOverlay(treeCid, status = Completed.some)).tryGet() + + ( + await repo.putCellCidsAndProofs( + treeCid, @[(0.Natural, cellCid, blk1.cid, proof1)] + ) + ).tryGet() + (await repo.putCidsAndProofs(treeCid, @[(1.Natural, blk2.cid, proof2)])).tryGet() + + let + leaf1 = (await repo.getLeafMetadata(treeCid, 0.Natural)).tryGet() + leaf2 = (await repo.getLeafMetadata(treeCid, 1.Natural)).tryGet() + + check: + leaf1.isCell == true + leaf1.cellCid == cellCid + leaf1.blkCid == blk1.cid + leaf2.isCell == false + + test "Empty blkCid skips refcount in cell and regular paths": + let + realBlk = createTestBlock(700) + cellCid1 = bt.Block.new("cell1".toBytes).tryGet().cid + cellCid2 = bt.Block.new("cell2".toBytes).tryGet().cid + emptyBlkCid = emptyCid(CIDv1, multiCodec("sha2-256"), BlockCodec).tryGet() + (_, tree) = makeManifestAndTree(@[realBlk, realBlk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof1 = tree.getProof(0).tryGet() + proof2 = tree.getProof(1).tryGet() + + (await repo.putOverlay(treeCid, status = Completed.some)).tryGet() + + ( + await repo.putCellCidsAndProofs( + treeCid, + @[ + (0.Natural, cellCid1, realBlk.cid, proof1), + (1.Natural, cellCid2, emptyBlkCid, proof2), + ], + ) + ).tryGet() + + let + leaf1 = (await repo.getLeafMetadata(treeCid, 0.Natural)).tryGet() + leaf2 = (await repo.getLeafMetadata(treeCid, 1.Natural)).tryGet() + realRefCountBefore = (await repo.blockRefCount(realBlk.cid)).tryGet() + emptyRefCountRes = await repo.blockRefCount(emptyBlkCid) + + check: + leaf1.blkCid == realBlk.cid + leaf1.blkCid.isEmpty == false + leaf2.isCell == true + leaf2.cellCid == cellCid2 + leaf2.blkCid == emptyBlkCid + leaf2.blkCid.isEmpty == true + realRefCountBefore == 1 + emptyRefCountRes.isErr + + (await repo.delLeafBlockMetadata(treeCid, @[0.Natural, 1.Natural])).tryGet() + + let refCountAfter = await repo.blockRefCount(realBlk.cid) + check refCountAfter.isErr + + let + (_, tree2) = makeManifestAndTree(@[realBlk]).tryGet() + treeCid2 = tree2.rootCid.tryGet() + proof3 = tree2.getProof(0).tryGet() + emptyBlk = bt.Block.new(newSeq[byte](0)).tryGet() + + check emptyBlk.cid.isEmpty == true + + (await repo.putOverlay(treeCid2, status = Completed.some)).tryGet() + (await repo.putBlocks(treeCid2, @[(emptyBlk, 0.Natural, proof3)])).tryGet() + + let leaf3 = (await repo.getLeafMetadata(treeCid2, 0.Natural)).tryGet() + check: + leaf3.blkCid.isEmpty == true + leaf3.isCell == false + + test "Should correctly handle multi-index delete with same block CID (refCount aggregation)": + let + innerRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 2000'nb) + blk = createTestBlock(256) + + let (_, tree) = makeManifestAndTree(@[blk, blk, blk, blk, blk, blk]).tryGet() + let treeCid = tree.rootCid.tryGet() + + var blocks = BitSeq.init(6) + blocks.setBit(1) + blocks.setBit(3) + blocks.setBit(5) + + ( + await innerRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = blocks + ) + ).tryGet() + + let + proof1 = tree.getProof(1).tryGet() + proof3 = tree.getProof(3).tryGet() + proof5 = tree.getProof(5).tryGet() + + ( + await innerRepo.putBlocks( + treeCid, + @[ + (blk, 1.Natural, proof1), (blk, 3.Natural, proof3), (blk, 5.Natural, proof5) + ], + ) + ).tryGet() + + check (await innerRepo.blockRefCount(blk.cid)).tryGet() == 3.Natural + check innerRepo.quotaUsedBytes == 256.NBytes + + ( + await innerRepo.delLeafBlockMetadata( + treeCid, @[1.Natural, 3.Natural, 5.Natural] + ) + ).tryGet() + + check not (await blk.cid in innerRepo) + check innerRepo.quotaUsedBytes == 0.NBytes + +proc runFsSqliteTests() = + let repoDir = createTempDir("archivist-", "-repostore") + + testCells( + "Cell handling FS+SQLite backend", + repoDsProvider = proc(): KVStore = + if not dirExists(repoDir): + createDir(repoDir) + let tp = Taskpool.new() + FSKVStore.new(repoDir, tp, depth = 5).tryGet(), + metaDsProvider = proc(): KVStore = + let tp = Taskpool.new() + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + after = proc(): Future[void] {.async.} = + os.removeDir(repoDir), + ) + +runFsSqliteTests() diff --git a/tests/archivist/stores/repostore/overlays/testconcurrent.nim b/tests/archivist/stores/repostore/overlays/testconcurrent.nim new file mode 100644 index 00000000..987d1069 --- /dev/null +++ b/tests/archivist/stores/repostore/overlays/testconcurrent.nim @@ -0,0 +1,281 @@ +import std/os +import std/tempfiles + +import pkg/questionable +import pkg/questionable/results + +import pkg/chronos +import pkg/stew/bitseqs +import pkg/kvstore +import pkg/kvstore/fsds +import pkg/taskpools + +import pkg/archivist/stores +import pkg/archivist/stores/repostore/operations +import pkg/archivist/stores/repostore/types +import pkg/archivist/clock + +import pkg/archivist/merkletree/archivist + +import ../../../../asynctest +import ../../../helpers +import ../../../helpers/mockclock +import ../../../examples + +import ./helpers + +type KVStoreProvider* = proc(): KVStore {.gcsafe.} + +proc testConcurrent*( + name: string, + repoDsProvider: KVStoreProvider, + metaDsProvider: KVStoreProvider, + before: Before = nil, + after: After = nil, +) = + suite name: + var + repoDs: KVStore + metaDs: KVStore + mockClock: MockClock + repo: RepoStore + + let now: SecondsSince1970 = 1000 + + setup: + if not isNil(before): + await before() + repoDs = repoDsProvider() + metaDs = metaDsProvider() + mockClock = MockClock.new() + mockClock.set(now) + repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = + 10000'nb) + + teardown: + (await repoDs.close()).tryGet + (await metaDs.close()).tryGet + if not isNil(after): + await after() + + test "Concurrent put operations on same overlay are serialized": + let + blk1 = createTestBlock(800) + blk2 = createTestBlock(801) + blk3 = createTestBlock(802) + (_, tree) = makeManifestAndTree(@[blk1, blk2, blk3]).tryGet() + treeCid = tree.rootCid.tryGet() + proof1 = tree.getProof(0).tryGet() + proof2 = tree.getProof(1).tryGet() + proof3 = tree.getProof(2).tryGet() + + (await repo.putOverlay(treeCid, status = Completed.some)).tryGet() + + let + put1Future = repo.putBlocks(treeCid, @[(blk1, 0.Natural, proof1)]) + put2Future = repo.putBlocks(treeCid, @[(blk2, 1.Natural, proof2)]) + put3Future = repo.putBlocks(treeCid, @[(blk3, 2.Natural, proof3)]) + + await allFutures(@[put1Future, put2Future, put3Future]) + + check (await put1Future).isOk + check (await put2Future).isOk + check (await put3Future).isOk + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check: + meta.blocks[0] == true + meta.blocks[1] == true + meta.blocks[2] == true + + let inconsistencies = (await repo.verifyBlockBitState(treeCid)).tryGet() + check inconsistencies.len == 0 + + test "Sequential CAS updates handle retries correctly": + let + blk1 = createTestBlock(840) + blk2 = createTestBlock(841) + blk3 = createTestBlock(842) + (_, tree) = makeManifestAndTree(@[blk1, blk2, blk3]).tryGet() + treeCid = tree.rootCid.tryGet() + proof1 = tree.getProof(0).tryGet() + proof2 = tree.getProof(1).tryGet() + proof3 = tree.getProof(2).tryGet() + + (await repo.putOverlay(treeCid, status = Completed.some)).tryGet() + + for i in 0 ..< 3: + let blk = + if i == 0: + blk1 + elif i == 1: + blk2 + else: + blk3 + let prf = + if i == 0: + proof1 + elif i == 1: + proof2 + else: + proof3 + + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, prf)])).tryGet() + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.blocks[0] == true + + let inconsistencies = (await repo.verifyBlockBitState(treeCid)).tryGet() + check inconsistencies.len == 0 + + test "Concurrent overlay metadata updates preserve latest values": + let treeCid = Cid.example + + let + update1 = repo.putOverlay( + treeCid, status = Storing.some, blocks = BitSeq.init(1), expiry = 100 + ) + update2 = repo.putOverlay( + treeCid, status = Completed.some, blocks = BitSeq.init(2), expiry = 200 + ) + + let + res1 = await update1 + res2 = await update2 + + check res1.isOk or res2.isOk + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.status in {Storing, Completed} + + test "Concurrent putBlocks and dropOverlay on different overlays": + let + blk1 = createTestBlock(900) + blk2 = createTestBlock(901) + blk3 = createTestBlock(902) + blk4 = createTestBlock(903) + (_, treeA) = makeManifestAndTree(@[blk1, blk2]).tryGet() + (_, treeB) = makeManifestAndTree(@[blk3, blk4]).tryGet() + treeCidA = treeA.rootCid.tryGet() + treeCidB = treeB.rootCid.tryGet() + proofA1 = treeA.getProof(0).tryGet() + proofA2 = treeA.getProof(1).tryGet() + proofB1 = treeB.getProof(0).tryGet() + proofB2 = treeB.getProof(1).tryGet() + + ( + await repo.putOverlay( + treeCid = treeCidA, status = Completed.some, blocks = BitSeq.init(2) + ) + ).tryGet() + + ( + await repo.putOverlay( + treeCid = treeCidB, status = Completed.some, blocks = BitSeq.init(2) + ) + ).tryGet() + + (await repo.putBlocks(treeCidB, @[(blk3, 0.Natural, proofB1)])).tryGet() + (await repo.putBlocks(treeCidB, @[(blk4, 1.Natural, proofB2)])).tryGet() + + let + putFuture = repo.putBlocks( + treeCidA, @[(blk1, 0.Natural, proofA1), (blk2, 1.Natural, proofA2)] + ) + dropFuture = repo.dropOverlay(treeCidB) + + await allFutures(@[putFuture, dropFuture]) + + check (await putFuture).isOk + check (await dropFuture).isOk + + let metaA = (await repo.getOverlay(treeCidA)).tryGet() + check: + metaA.blocks[0] == true + metaA.blocks[1] == true + + let inconsistenciesA = (await repo.verifyBlockBitState(treeCidA)).tryGet() + check inconsistenciesA.len == 0 + + check (await repo.getOverlay(treeCidB)).isErr + check (await repo.getBlock(blk3.cid)).isErr + check (await repo.getBlock(blk4.cid)).isErr + + test "putBlocks aborts when delete started first": + let + blk1 = createTestBlock(910) + blk2 = createTestBlock(911) + blk3 = createTestBlock(912) + (_, tree) = makeManifestAndTree(@[blk1, blk2, blk3]).tryGet() + treeCid = tree.rootCid.tryGet() + proof1 = tree.getProof(0).tryGet() + proof2 = tree.getProof(1).tryGet() + + ( + await repo.putOverlay( + treeCid = treeCid, status = Deleting.some, blocks = BitSeq.init(3) + ) + ).tryGet() + + let putResult = await repo.putBlocks( + treeCid, @[(blk1, 0.Natural, proof1), (blk2, 1.Natural, proof2)] + ) + + check putResult.isErr + check putResult.error() of OverlayDeletingError + + test "putBlocks aborts when delete starts mid-operation": + let + blk1 = createTestBlock(920) + blk2 = createTestBlock(921) + blk3 = createTestBlock(922) + (_, tree) = makeManifestAndTree(@[blk1, blk2, blk3]).tryGet() + treeCid = tree.rootCid.tryGet() + proof1 = tree.getProof(0).tryGet() + proof2 = tree.getProof(1).tryGet() + proof3 = tree.getProof(2).tryGet() + + ( + await repo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = BitSeq.init(3) + ) + ).tryGet() + + (await repo.putBlocks(treeCid, @[(blk1, 0.Natural, proof1)])).tryGet() + + let + putFuture = repo.putBlocks( + treeCid, @[(blk2, 1.Natural, proof2), (blk3, 2.Natural, proof3)] + ) + dropFuture = repo.dropOverlay(treeCid) + + await allFutures(@[putFuture, dropFuture]) + + let putResult = await putFuture + let dropResult = await dropFuture + + check dropResult.isOk + + if putResult.isErr: + check putResult.error() of OverlayDeletingError + + check (await repo.getOverlay(treeCid)).isErr + +proc runFsSqliteTests() = + let repoDir = createTempDir("archivist-", "-repostore") + + testConcurrent( + "Concurrent overlay updates FS+SQLite backend", + repoDsProvider = proc(): KVStore = + if not dirExists(repoDir): + createDir(repoDir) + let tp = Taskpool.new() + FSKVStore.new(repoDir, tp, depth = 5).tryGet(), + metaDsProvider = proc(): KVStore = + let tp = Taskpool.new() + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + after = proc(): Future[void] {.async.} = + os.removeDir(repoDir), + ) + +runFsSqliteTests() diff --git a/tests/archivist/stores/repostore/overlays/testlifecycle.nim b/tests/archivist/stores/repostore/overlays/testlifecycle.nim new file mode 100644 index 00000000..56946e12 --- /dev/null +++ b/tests/archivist/stores/repostore/overlays/testlifecycle.nim @@ -0,0 +1,688 @@ +import std/os +import std/tempfiles + +import pkg/questionable +import pkg/questionable/results + +import pkg/chronos +import pkg/taskpools +import pkg/stew/bitseqs +import pkg/kvstore +import pkg/kvstore/fsds + +import pkg/archivist/stores +import pkg/archivist/stores/repostore/operations +import pkg/archivist/stores/repostore/types +import pkg/archivist/blocktype as bt +import pkg/archivist/clock + +import pkg/archivist/merkletree/archivist + +import ../../../../asynctest +import ../../../helpers +import ../../../helpers/mockclock +import ../../../examples + +import ./helpers + +proc testLifecycle*( + name: string, + repoDsProvider: KVStoreProvider, + metaDsProvider: KVStoreProvider, + before: Before = nil, + after: After = nil, +) = + suite name: + var + repoDs: KVStore + metaDs: KVStore + mockClock: MockClock + repo: RepoStore + + let now: SecondsSince1970 = 123 + + setup: + if not isNil(before): + await before() + repoDs = repoDsProvider() + metaDs = metaDsProvider() + mockClock = MockClock.new() + mockClock.set(now) + repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = + 2000'nb) + + teardown: + (await repoDs.close()).tryGet + (await metaDs.close()).tryGet + if not isNil(after): + await after() + + test "Should drop overlay and remove leafs, blocks, and metadata": + let blk = createTestBlock(120) + let (treeCid, index) = (await putBlockWithOverlay(repo, blk)).tryGet() + + (await repo.dropOverlay(treeCid)).tryGet() + + let leafResult = await repo.getLeafMetadata(treeCid, index) + let blockResult = await repo.getBlock(blk.cid) + let overlayResult = await repo.getOverlay(treeCid) + + check leafResult.isErr + check leafResult.error() of BlockNotFoundError + check blockResult.isErr + check blockResult.error() of BlockNotFoundError + check overlayResult.isErr + check overlayResult.error() of KVStoreKeyNotFound + + test "Should only decrement refcount for shared blocks when one overlay is dropped": + let + shared = createTestBlock(121) + extra1 = createTestBlock(122) + extra2 = createTestBlock(123) + (_, tree1) = makeManifestAndTree(@[shared, extra1]).tryGet() + (_, tree2) = makeManifestAndTree(@[extra2, shared]).tryGet() + treeCid1 = tree1.rootCid.tryGet() + treeCid2 = tree2.rootCid.tryGet() + proof1 = tree1.getProof(0).tryGet() + proof2 = tree2.getProof(1).tryGet() + + ( + await repo.putOverlay( + treeCid = treeCid1, status = Completed.some, blocks = BitSeq.init(2) + ) + ).tryGet() + + ( + await repo.putOverlay( + treeCid = treeCid2, status = Completed.some, blocks = BitSeq.init(2) + ) + ).tryGet() + + (await repo.putBlocks(treeCid1, @[(shared, 0.Natural, proof1)])).tryGet() + (await repo.putBlocks(treeCid2, @[(shared, 1.Natural, proof2)])).tryGet() + check (await repo.blockRefCount(shared.cid)).tryGet() == 2.Natural + + (await repo.dropOverlay(treeCid1)).tryGet() + + check (await repo.blockRefCount(shared.cid)).tryGet() == 1.Natural + check (await repo.getBlock(shared.cid)).isOk + check (await repo.getOverlay(treeCid1)).isErr + + test "Should drop empty overlay metadata": + let treeCid = Cid.example + + ( + await repo.putOverlay( + treeCid = treeCid, status = Storing.some, blocks = BitSeq.init(0) + ) + ).tryGet() + + (await repo.dropOverlay(treeCid)).tryGet() + + let res = await repo.getOverlay(treeCid) + check res.isErr + check res.error() of KVStoreKeyNotFound + + test "Should finalize overlay by moving metadata and leaves": + let + blk = createTestBlock(124) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + realTreeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + tmpCid = (await repo.createTmpOverlay()).tryGet() + + (await repo.putBlocks(tmpCid, @[(blk, 0.Natural, proof)])).tryGet() + (await repo.finalizeOverlay(tmpCid, realTreeCid)).tryGet() + + let + realOverlay = (await repo.getOverlay(realTreeCid)).tryGet() + leaf = (await repo.getLeafMetadata(realTreeCid, 0.Natural)).tryGet() + + check realOverlay.status == Storing + check realOverlay.blocks[0] + check leaf.blkCid == blk.cid + + test "Should remove old tmp overlay metadata after finalize": + let + blk = createTestBlock(125) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + realTreeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + tmpCid = (await repo.createTmpOverlay()).tryGet() + + (await repo.putBlocks(tmpCid, @[(blk, 0.Natural, proof)])).tryGet() + (await repo.finalizeOverlay(tmpCid, realTreeCid)).tryGet() + + let res = await repo.getOverlay(tmpCid) + check res.isErr + check res.error() of KVStoreKeyNotFound + + test "Should provide block and proof under new tree after finalize": + let + blk = createTestBlock(126) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + realTreeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + tmpCid = (await repo.createTmpOverlay()).tryGet() + + (await repo.putBlocks(tmpCid, @[(blk, 0.Natural, proof)])).tryGet() + (await repo.finalizeOverlay(tmpCid, realTreeCid)).tryGet() + + let (_, gotBlock, gotProof) = + (await repo.getBlockAndProof(realTreeCid, 0.Natural)).tryGet() + check gotBlock.cid == blk.cid + check gotProof.index == proof.index + check gotProof.nleaves == proof.nleaves + + test "finalizeOverlay succeeds when destination leaf exists (content-addressed idempotency)": + let + blk = createTestBlock(127) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + realTreeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + tmpCid = (await repo.createTmpOverlay()).tryGet() + + # Pre-populate destination with the SAME block at same index + (await repo.putOverlay(realTreeCid, status = Storing.some)).tryGet() + (await repo.putBlocks(realTreeCid, @[(blk, 0.Natural, proof)])).tryGet() + + # Put same data in tmp overlay + (await repo.putBlocks(tmpCid, @[(blk, 0.Natural, proof)])).tryGet() + + # Finalize should succeed (idempotent operation) + let res = await repo.finalizeOverlay(tmpCid, realTreeCid) + check res.isOk + + # Tmp overlay should be gone (dropped cleanly) + let tmpMetaRes = await repo.getOverlay(tmpCid) + check tmpMetaRes.isErr + + # Destination should still have the data + let realLeaf = (await repo.getLeafMetadata(realTreeCid, 0.Natural)).tryGet() + check realLeaf.blkCid == blk.cid + + test "finalizeOverlay handles conflict with different data at same index": + let + blk = createTestBlock(128) + existingBlk = createTestBlock(129) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + realTreeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + tmpCid = (await repo.createTmpOverlay()).tryGet() + + # Pre-populate destination with a different block at same index + (await repo.putOverlay(realTreeCid, status = Storing.some)).tryGet() + let + (_, existingTree) = makeManifestAndTree(@[existingBlk]).tryGet() + existingProof = existingTree.getProof(0).tryGet() + + (await repo.putBlocks(realTreeCid, @[(existingBlk, 0.Natural, existingProof)])).tryGet() + + # Put different data in tmp overlay + (await repo.putBlocks(tmpCid, @[(blk, 0.Natural, proof)])).tryGet() + + # Finalize should succeed (KVConflictError is caught and handled) + let res = await repo.finalizeOverlay(tmpCid, realTreeCid) + check res.isOk + + # Tmp overlay should be gone (dropped on conflict) + let tmpMetaRes = await repo.getOverlay(tmpCid) + check tmpMetaRes.isErr + + # Destination should be unchanged (original data preserved) + let realLeaf = (await repo.getLeafMetadata(realTreeCid, 0.Natural)).tryGet() + check realLeaf.blkCid == existingBlk.cid + + test "finalizeOverlay succeeds when destination metadata exists": + let + blk = createTestBlock(129) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + realTreeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + tmpCid = (await repo.createTmpOverlay()).tryGet() + + # Create destination with metadata but no leaf data + (await repo.putOverlay(realTreeCid, status = Completed.some)).tryGet() + + (await repo.putBlocks(tmpCid, @[(blk, 0.Natural, proof)])).tryGet() + let res = await repo.finalizeOverlay(tmpCid, realTreeCid) + + check res.isOk + + test "BitSeq preserved through finalizeOverlay (temp to real)": + let + blk1 = createTestBlock(300) + blk2 = createTestBlock(301) + (_, tree) = makeManifestAndTree(@[blk1, blk2]).tryGet() + realTreeCid = tree.rootCid.tryGet() + proof1 = tree.getProof(0).tryGet() + proof2 = tree.getProof(1).tryGet() + var capturedTmpCid: Cid + + let res = await repo.withTmpOverlay( + body = proc( + tmpCid: Cid + ): Future[?!Cid] {.closure, async: (raises: [CancelledError]).} = + capturedTmpCid = tmpCid + ?await repo.putBlocks( + tmpCid, @[(blk1, 0.Natural, proof1), (blk2, 1.Natural, proof2)] + ) + success(realTreeCid) + ) + + check res.isOk + + let realMeta = (await repo.getOverlay(realTreeCid)).tryGet() + check realMeta.blocks.len >= 2 + check realMeta.blocks[0] == true + check realMeta.blocks[1] == true + + let inconsistencies = (await repo.verifyBlockBitState(realTreeCid)).tryGet() + check inconsistencies.len == 0 + + let tmpMetaRes = await repo.getOverlay(capturedTmpCid) + check tmpMetaRes.isErr + + test "BitSeq correct after re-insert of deleted block (resurrection)": + let + blk = createTestBlock(302) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + (await repo.putOverlay(treeCid, status = Completed.some)).tryGet() + + # Insert + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() + let meta1 = (await repo.getOverlay(treeCid)).tryGet() + check meta1.blocks[0] == true + + # Delete + (await repo.delLeafBlockMetadata(treeCid, @[0.Natural])).tryGet() + let meta2 = (await repo.getOverlay(treeCid)).tryGet() + check meta2.blocks[0] == false + + # After deletion, overlay status is Deleting - no resurrection allowed + let meta3 = (await repo.getOverlay(treeCid)).tryGet() + check meta3.status == Deleting + check meta3.blocks[0] == false + + # Attempting to re-insert should fail with OverlayDeletingError + let putResult = await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)]) + check putResult.isErr + check putResult.error() of OverlayDeletingError + + let inconsistencies = (await repo.verifyBlockBitState(treeCid)).tryGet() + check inconsistencies.len == 0 + + test "Should clear leaf metadata when block is deleted from dataset": + let + dataset = (await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb)).tryGet + blk = dataset[0] + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var blocks = BitSeq.init(1) + + ( + await repo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = blocks + ) + ).tryGet() + + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() + + discard (await repo.getLeafMetadata(treeCid, 0.Natural)).tryGet() + + (await repo.delBlock(treeCid, 0.Natural)).tryGet() + + let err = (await repo.getLeafMetadata(treeCid, 0.Natural)).error() + check err of BlockNotFoundError + + test "Should fail re-put after delete due to Deleting status": + let + dataset = (await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb)).tryGet + blk = dataset[0] + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var blocks = BitSeq.init(1) + blocks.setBit(0) + + ( + await repo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = blocks + ) + ).tryGet() + + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() + + (await repo.delBlock(treeCid, 0.Natural)).tryGet() + + let putResult = await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)]) + check putResult.isErr + check putResult.error() of OverlayDeletingError + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.status == Deleting + + test "Should create overlay before running body": + let treeCid = Cid.example + var statusDuringBody = Failure + + let res = await repo.withOverlay( + treeCid, + status = Storing.some, + body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = + let meta = ?await repo.getOverlay(treeCid) + statusDuringBody = meta.status + success(), + ) + + check res.isOk + check statusDuringBody == Storing + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.status == Completed + + test "Should set completed state on async body success": + let treeCid = Cid.example + + let res = await repo.withOverlay( + treeCid, + status = Storing.some, + body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = + success(), + ) + + check res.isOk + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.status == Completed + + test "Should set failure state on async body failure": + let treeCid = Cid.example + + let res = await repo.withOverlay( + treeCid, + status = Storing.some, + body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = + failure(newException(ValueError, "body failed")), + ) + + check res.isErr + check res.error of ValueError + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.status == Failure + + test "Should preserve typed body result": + let treeCid = Cid.example + + let res = await repo.withOverlay( + treeCid, + status = Storing.some, + body = proc(): Future[?!int] {.closure, async: (raises: [CancelledError]).} = + success(42), + ) + + check res.isOk + check res.get == 42 + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.status == Completed + + test "Should use custom expiry for final overlay metadata": + let + treeCid = Cid.example + customExpiry: SecondsSince1970 = 500 + + let res = await repo.withOverlay( + treeCid, + expiry = customExpiry, + body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = + success(), + ) + + check res.isOk + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.status == Completed + check meta.expiry == customExpiry + + test "Should keep initial status when body is cancelled": + let treeCid = Cid.example + var bodyStarted = newFuture[void]("withOverlay.cancel.started") + + let op = repo.withOverlay( + treeCid, + status = Repairing.some, + body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = + if not bodyStarted.finished: + bodyStarted.complete() + await sleepAsync(10.seconds) + success(), + ) + + await bodyStarted.wait(500.millis) + await op.cancelAndWait() + + expect CancelledError: + discard await op + + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.status == Repairing + + test "Should retain written leaf metadata when body is cancelled": + let + blk = createTestBlock(132) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + var writeDone = newFuture[void]("withOverlay.cancel.writeDone") + + let op = repo.withOverlay( + treeCid, + status = Storing.some, + body = proc(): Future[?!void] {.closure, async: (raises: [CancelledError]).} = + ?await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)]) + if not writeDone.finished: + writeDone.complete() + await sleepAsync(10.seconds) + success(), + ) + + await writeDone.wait(500.millis) + await op.cancelAndWait() + + expect CancelledError: + discard await op + + let + meta = (await repo.getOverlay(treeCid)).tryGet() + leaf = (await repo.getLeafMetadata(treeCid, 0.Natural)).tryGet() + check meta.status == Storing + check leaf.blkCid == blk.cid + + test "Should create tmp overlay before running body": + let realTreeCid = Cid.example + var statusDuringBody = Failure + + let res = await repo.withTmpOverlay( + body = proc( + tmpCid: Cid + ): Future[?!Cid] {.closure, async: (raises: [CancelledError]).} = + let tmpMeta = ?await repo.getOverlay(tmpCid) + statusDuringBody = tmpMeta.status + success(realTreeCid) + ) + + check res.isOk + check res.get == realTreeCid + check statusDuringBody == Storing + + let realMeta = (await repo.getOverlay(realTreeCid)).tryGet() + check realMeta.status == Completed + + test "Should finalize tmp overlay and move leaves to real tree": + let + blk = createTestBlock(130) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + realTreeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + var capturedTmpCid: Cid + + let res = await repo.withTmpOverlay( + body = proc( + tmpCid: Cid + ): Future[?!Cid] {.closure, async: (raises: [CancelledError]).} = + capturedTmpCid = tmpCid + ?await repo.putBlocks(tmpCid, @[(blk, 0.Natural, proof)]) + success(realTreeCid) + ) + + check res.isOk + check res.get == realTreeCid + + let leaf = (await repo.getLeafMetadata(realTreeCid, 0.Natural)).tryGet() + check leaf.blkCid == blk.cid + + let tmpMetaRes = await repo.getOverlay(capturedTmpCid) + check tmpMetaRes.isErr + check tmpMetaRes.error() of KVStoreKeyNotFound + + let realMeta = (await repo.getOverlay(realTreeCid)).tryGet() + check realMeta.status == Completed + + test "Should drop tmp overlay metadata on body failure": + var capturedTmpCid: Cid + + let res = await repo.withTmpOverlay( + body = proc( + tmpCid: Cid + ): Future[?!Cid] {.closure, async: (raises: [CancelledError]).} = + capturedTmpCid = tmpCid + Cid.failure("encode failed") + ) + + check res.isErr + check "encode failed" in res.error.msg + + let tmpMetaRes = await repo.getOverlay(capturedTmpCid) + check tmpMetaRes.isErr + check tmpMetaRes.error() of KVStoreKeyNotFound + + test "Should drop tmp overlay and cleanup stored leafs on body failure": + let + blk = createTestBlock(131) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + proof = tree.getProof(0).tryGet() + var capturedTmpCid: Cid + + let res = await repo.withTmpOverlay( + body = proc( + tmpCid: Cid + ): Future[?!Cid] {.closure, async: (raises: [CancelledError]).} = + capturedTmpCid = tmpCid + ?await repo.putBlocks(tmpCid, @[(blk, 0.Natural, proof)]) + Cid.failure("encode failed after storing") + ) + + check res.isErr + check "encode failed after storing" in res.error.msg + + let tmpMetaRes = await repo.getOverlay(capturedTmpCid) + check tmpMetaRes.isErr + check tmpMetaRes.error() of KVStoreKeyNotFound + + let tmpLeafRes = await repo.getLeafMetadata(capturedTmpCid, 0.Natural) + check tmpLeafRes.isErr + check tmpLeafRes.error() of BlockNotFoundError + + test "Should drop tmp overlay metadata when body is cancelled": + let realTreeCid = Cid.example + var + capturedTmpCid: Cid + bodyStarted = newFuture[void]("withTmpOverlay.cancel.started") + + let op = repo.withTmpOverlay( + body = proc( + tmpCid: Cid + ): Future[?!Cid] {.closure, async: (raises: [CancelledError]).} = + capturedTmpCid = tmpCid + if not bodyStarted.finished: + bodyStarted.complete() + await sleepAsync(10.seconds) + success(realTreeCid) + ) + + await bodyStarted.wait(500.millis) + await op.cancelAndWait() + + expect CancelledError: + discard await op + + let tmpMetaRes = await repo.getOverlay(capturedTmpCid) + check tmpMetaRes.isErr + check tmpMetaRes.error() of KVStoreKeyNotFound + + test "Should cleanup tmp overlay leaf and block on body cancellation": + let + blk = createTestBlock(133) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + proof = tree.getProof(0).tryGet() + + var + capturedTmpCid: Cid + writeDone = newFuture[void]("withTmpOverlay.cancel.writeDone") + + let op = repo.withTmpOverlay( + body = proc( + tmpCid: Cid + ): Future[?!Cid] {.closure, async: (raises: [CancelledError]).} = + capturedTmpCid = tmpCid + ?await repo.putBlocks(tmpCid, @[(blk, 0.Natural, proof)]) + if not writeDone.finished: + writeDone.complete() + await sleepAsync(10.seconds) + success(tmpCid) + ) + + await writeDone.wait(500.millis) + await op.cancelAndWait() + + expect CancelledError: + discard await op + + let tmpMetaRes = await repo.getOverlay(capturedTmpCid) + check tmpMetaRes.isErr + check tmpMetaRes.error() of KVStoreKeyNotFound + + let tmpLeafRes = await repo.getLeafMetadata(capturedTmpCid, 0.Natural) + check tmpLeafRes.isErr + check tmpLeafRes.error() of BlockNotFoundError + + let blkRes = await repo.getBlock(blk.cid) + check blkRes.isErr + check blkRes.error() of BlockNotFoundError + +proc runFsSqliteTests() = + let repoDir = createTempDir("archivist-", "-repostore") + + testLifecycle( + "Overlay lifecycle FS+SQLite backend", + repoDsProvider = proc(): KVStore = + if not dirExists(repoDir): + createDir(repoDir) + let tp = Taskpool.new() + FSKVStore.new(repoDir, tp, depth = 5).tryGet(), + metaDsProvider = proc(): KVStore = + let tp = Taskpool.new() + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + after = proc(): Future[void] {.async.} = + os.removeDir(repoDir), + ) + +runFsSqliteTests() diff --git a/tests/archivist/stores/repostore/overlays/testmetadata.nim b/tests/archivist/stores/repostore/overlays/testmetadata.nim new file mode 100644 index 00000000..cc17f163 --- /dev/null +++ b/tests/archivist/stores/repostore/overlays/testmetadata.nim @@ -0,0 +1,356 @@ +import std/os +import std/tempfiles +import std/sequtils + +import pkg/questionable +import pkg/questionable/results + +import pkg/chronos +import pkg/taskpools +import pkg/stew/bitseqs +import pkg/kvstore +import pkg/kvstore/fsds + +import pkg/archivist/stores +import pkg/archivist/stores/repostore/operations +import pkg/archivist/stores/repostore/overlays/coders +import pkg/archivist/stores/repostore/types +import pkg/archivist/blocktype as bt +import pkg/archivist/clock +import pkg/archivist/utils + +import pkg/archivist/merkletree/archivist + +import ../../../../asynctest +import ../../../helpers +import ../../../helpers/mockclock +import ../../../examples + +import ./helpers + +proc testMetadata*( + name: string, + repoDsProvider: KVStoreProvider, + metaDsProvider: KVStoreProvider, + before: Before = nil, + after: After = nil, +) = + asyncchecksuite name: + var + repoDs: KVStore + metaDs: KVStore + mockClock: MockClock + repo: RepoStore + + let now: SecondsSince1970 = 123 + + setup: + if not isNil(before): + await before() + repoDs = repoDsProvider() + metaDs = metaDsProvider() + mockClock = MockClock.new() + mockClock.set(now) + repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = + 2000'nb) + + teardown: + (await repoDs.close()).tryGet + (await metaDs.close()).tryGet + if not isNil(after): + await after() + + test "Should put and get overlay metadata": + let treeCid = Cid.example + + var bits = BitSeq.init(10) + bits.setBit(0) + bits.setBit(9) + + let meta = OverlayMetadata(status: Completed, expiry: now + 100, blocks: bits) + + ( + await repo.putOverlay( + treeCid, status = meta.status.some, blocks = meta.blocks, expiry = meta.expiry + ) + ).tryGet() + let got = (await repo.getOverlay(treeCid)).tryGet() + + check got.status == meta.status + check got.expiry == meta.expiry + check got.blocks == meta.blocks + + test "Should update existing overlay metadata": + let treeCid = Cid.example + + let meta1 = + OverlayMetadata(status: Storing, expiry: now + 1, blocks: BitSeq.init(1)) + + ( + await repo.putOverlay( + treeCid, + status = meta1.status.some, + blocks = meta1.blocks, + expiry = meta1.expiry, + ) + ).tryGet() + + var bits = BitSeq.init(2) + bits.setBit(1) + let meta2 = OverlayMetadata(status: Completed, expiry: now + 2, blocks: bits) + + ( + await repo.putOverlay( + treeCid, + status = meta2.status.some, + blocks = meta2.blocks, + expiry = meta2.expiry, + ) + ).tryGet() + + let got = (await repo.getOverlay(treeCid)).tryGet() + check got.status == meta2.status + check got.expiry == meta2.expiry + check got.blocks == meta2.blocks + + test "Should delete overlay metadata": + let treeCid = Cid.example + let meta = + OverlayMetadata(status: Completed, expiry: now + 10, blocks: BitSeq.init(0)) + + ( + await repo.putOverlay( + treeCid, status = meta.status.some, blocks = meta.blocks, expiry = meta.expiry + ) + ).tryGet() + (await repo.deleteOverlay(treeCid)).tryGet() + + let res = await repo.getOverlay(treeCid) + check res.isErr + check res.error() of KVStoreKeyNotFound + + test "Should fail to delete non-existent overlay": + let res = await repo.deleteOverlay(Cid.example) + check res.isErr + + test "Should fail get for non-existent overlay with KVStoreKeyNotFound": + let res = await repo.getOverlay(Cid.example) + check res.isErr + check res.error() of KVStoreKeyNotFound + + test "hasBlock behavior across states": + let + blk = createTestBlock(100) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + (await repo.putOverlay(treeCid, status = Completed.some)).tryGet() + + # Not set, no block + check (await repo.hasBlock(treeCid, 0.Natural)).tryGet() == false + + # Set with block + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() + check (await repo.hasBlock(treeCid, 0.Natural)).tryGet() == true + + # Out of range (fast-path, no store hit) + check (await repo.hasBlock(treeCid, 10.Natural)).tryGet() == false + let leafRes = await repo.getLeafMetadata(treeCid, 10.Natural) + check leafRes.isErr + check leafRes.error() of BlockNotFoundError + + test "Shared blocks across overlays maintain correct BitSeq": + let + shared = createTestBlock(200) + (_, tree1) = makeManifestAndTree(@[shared]).tryGet() + treeCid1 = tree1.rootCid.tryGet() + proof1 = tree1.getProof(0).tryGet() + + (await repo.putOverlay(treeCid1, status = Completed.some)).tryGet() + (await repo.putBlocks(treeCid1, @[(shared, 0.Natural, proof1)])).tryGet() + + let meta1 = (await repo.getOverlay(treeCid1)).tryGet() + check meta1.blocks[0] == true + + let refCount = (await repo.blockRefCount(shared.cid)).tryGet() + check refCount == 1.Natural + + (await repo.dropOverlay(treeCid1)).tryGet() + + let blkRes = await repo.getBlock(shared.cid) + check blkRes.isErr + + test "Should handle non-contiguous indices in putBlocks (BitSeq length fix)": + let + innerRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 2000'nb) + dataset = + (await makeRandomBlocks(datasetSize = 2560, blockSize = 256'nb)).tryGet + blk = dataset[0] + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + # Create overlay with 10 blocks, but only insert at index 5 + var blocks = BitSeq.init(10) + + ( + await innerRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = blocks + ) + ).tryGet() + + # Put only index 5 (non-contiguous) + let proof = tree.getProof(5).tryGet() + (await innerRepo.putBlocks(treeCid, @[(blk, 5.Natural, proof)])).tryGet() + + # Verify block was stored + check (await innerRepo.getBlock(blk.cid)).isOk + check innerRepo.quotaUsedBytes == 256.NBytes + + # Verify overlay has correct bit set + let overlayMeta = (await innerRepo.getOverlay(treeCid)).tryGet() + check overlayMeta.blocks.len == 10 + check overlayMeta.blocks[5] == true + + test "Should create unique tmp overlays": + let tmpCid1 = (await repo.createTmpOverlay()).tryGet() + let tmpCid2 = (await repo.createTmpOverlay()).tryGet() + + check tmpCid1 != tmpCid2 + + test "Should create tmp overlay with storing status": + let tmpCid = (await repo.createTmpOverlay()).tryGet() + let meta = (await repo.getOverlay(tmpCid)).tryGet() + + check meta.status == Storing + + test "Should set tmp overlay expiry from clock and ttl": + let tmpCid = (await repo.createTmpOverlay()).tryGet() + let meta = (await repo.getOverlay(tmpCid)).tryGet() + + check meta.expiry == now + DefaultOverlayTtl + + test "Should list all overlays": + let + cid1 = createTestBlock(21).cid + cid2 = createTestBlock(22).cid + cid3 = createTestBlock(23).cid + + for (cid, status) in [(cid1, Completed), (cid2, Storing), (cid3, Failure)]: + ( + await repo.putOverlay( + treeCid = cid, status = status.some, blocks = BitSeq.init(1) + ) + ).tryGet() + + let iter = (await repo.listOverlays()).tryGet() + let cids = (await utils.collect(iter)).tryGet() + + check cids.len == 3 + check cids.anyIt(it == cid1) + check cids.anyIt(it == cid2) + check cids.anyIt(it == cid3) + + test "Should return empty list when no overlays exist": + let iter = (await repo.listOverlays()).tryGet() + let cids = (await utils.collect(iter)).tryGet() + + check cids.len == 0 + + test "Should filter overlays by state": + let + cid1 = createTestBlock(31).cid + cid2 = createTestBlock(32).cid + cid3 = createTestBlock(33).cid + + for (cid, status) in [(cid1, Completed), (cid2, Storing), (cid3, Completed)]: + ( + await repo.putOverlay( + treeCid = cid, status = status.some, blocks = BitSeq.init(1) + ) + ).tryGet() + + let iter = (await repo.listOverlaysInState(Completed)).tryGet() + let cids = (await utils.collect(iter)).tryGet() + + check cids.len == 2 + check cids.anyIt(it == cid1) + check cids.anyIt(it == cid3) + check cids.allIt(it != cid2) + + test "Should return empty list when state has no matches": + let cid = createTestBlock(41).cid + + ( + await repo.putOverlay( + treeCid = cid, status = Completed.some, blocks = BitSeq.init(1) + ) + ).tryGet() + + let iter = (await repo.listOverlaysInState(Deleting)).tryGet() + let cids = (await utils.collect(iter)).tryGet() + + check cids.len == 0 + + test "Should list overlays sorted by expiry ascending": + let + cid1 = createTestBlock(51).cid + cid2 = createTestBlock(52).cid + cid3 = createTestBlock(53).cid + + for (cid, status) in [ + (cid1, Storing.some), (cid2, Storing.some), (cid3, Storing.some) + ]: + ( + await repo.putOverlay( + treeCid = cid, status = status, blocks = BitSeq.init(1), expiry = 50 + ) + ).tryGet() + + let overlays = (await repo.listOverlaysByExpiry(limit = 10, offset = 0)).tryGet() + let overlayCids = overlays.mapIt(it[0]) + + check overlays.len == 3 + check overlayCids == @[cid2, cid3, cid1] + + test "Should apply limit and offset for expiry listing": + let + cid1 = createTestBlock(61).cid + cid2 = createTestBlock(62).cid + cid3 = createTestBlock(63).cid + cid4 = createTestBlock(64).cid + + for cid in [cid1, cid2, cid3, cid4]: + ( + await repo.putOverlay( + treeCid = cid, status = Completed.some, blocks = BitSeq.init(1) + ) + ).tryGet() + + let firstPage = (await repo.listOverlaysByExpiry(limit = 2, offset = 0)).tryGet() + let secondPage = (await repo.listOverlaysByExpiry(limit = 2, offset = 2)).tryGet() + + check firstPage.len == 2 + check secondPage.len == 2 + for (cid, _) in firstPage: + check secondPage.allIt(it[0] != cid) + +proc runFsSqliteTests() = + let repoDir = createTempDir("archivist-", "-repostore") + + testMetadata( + "Overlay metadata FS+SQLite backend", + repoDsProvider = proc(): KVStore = + if not dirExists(repoDir): + createDir(repoDir) + let tp = Taskpool.new() + FSKVStore.new(repoDir, tp, depth = 5).tryGet(), + metaDsProvider = proc(): KVStore = + let tp = Taskpool.new() + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + after = proc(): Future[void] {.async.} = + os.removeDir(repoDir), + ) + +runFsSqliteTests() diff --git a/tests/archivist/stores/repostore/testcoders.nim b/tests/archivist/stores/repostore/testcoders.nim index 690e89a6..2f54df1d 100644 --- a/tests/archivist/stores/repostore/testcoders.nim +++ b/tests/archivist/stores/repostore/testcoders.nim @@ -2,16 +2,21 @@ import std/random import pkg/unittest2 import pkg/stew/objects +import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results import pkg/archivist/clock +import pkg/archivist/merkletree +import pkg/archivist/blocktype as bt import pkg/archivist/stores/repostore/types import pkg/archivist/stores/repostore/coders import ../../helpers +import ../../examples +import ../../merkletree/helpers as mhelpers -suite "Test coders": +suite "Test repostore coders": proc rand(T: type NBytes): T = rand(Natural).NBytes @@ -22,16 +27,11 @@ suite "Test coders": proc rand(T: type QuotaUsage): T = QuotaUsage(used: rand(NBytes), reserved: rand(NBytes)) - proc rand(T: type BlockMetadata): T = - BlockMetadata( - expiry: rand(SecondsSince1970), size: rand(NBytes), refCount: rand(Natural) - ) - - proc rand(T: type DeleteResult): T = - DeleteResult(kind: rand(DeleteResultKind), released: rand(NBytes)) + proc rand(T: type Cid): T = + Cid.example - proc rand(T: type StoreResult): T = - StoreResult(kind: rand(StoreResultKind), used: rand(NBytes)) + proc rand(T: type BlockMetadata): T = + BlockMetadata(cid: rand(Cid), refCount: rand(Natural)) test "Natural encode/decode": for val in newSeqWith(100, rand(Natural)) & @[Natural.low, Natural.high]: @@ -48,12 +48,53 @@ suite "Test coders": check: success(val) == BlockMetadata.decode(encode(val)) - test "DeleteResult encode/decode": - for val in newSeqWith(100, rand(DeleteResult)): - check: - success(val) == DeleteResult.decode(encode(val)) + test "LeafMetadata encode/decode": + let + nodes = @[newSeqWith(32, rand(byte)), newSeqWith(32, rand(byte))] + proof = ArchivistProof.init(index = 0, nleaves = 4, nodes = nodes).tryGet() + val = LeafMetadata(deleted: false, blkCid: Cid.example, proof: proof) + decoded = LeafMetadata.decode(encode(val)).tryGet() - test "StoreResult encode/decode": - for val in newSeqWith(100, rand(StoreResult)): - check: - success(val) == StoreResult.decode(encode(val)) + check: + decoded.deleted == val.deleted + decoded.blkCid == val.blkCid + decoded.proof == val.proof + + test "LeafMetadata encode/decode with nil proof": + let + val = LeafMetadata(deleted: true, blkCid: Cid.example, proof: nil) + decoded = LeafMetadata.decode(encode(val)).tryGet() + + check: + decoded.deleted == val.deleted + decoded.blkCid == val.blkCid + decoded.proof.isNil + + test "LeafMetadata encode/decode with cell variant": + # Create two different CIDs using blocks + let + blkCid = bt.Block.new("block data".toBytes).tryGet().cid + cellCid = bt.Block.new("cell data".toBytes).tryGet().cid + nodes = @[newSeqWith(32, rand(byte)), newSeqWith(32, rand(byte))] + proof = ArchivistProof.init(index = 1, nleaves = 8, nodes = nodes).tryGet() + val = LeafMetadata( + deleted: false, blkCid: blkCid, proof: proof, isCell: true, cellCid: cellCid + ) + decoded = LeafMetadata.decode(encode(val)).tryGet() + + check: + decoded.deleted == val.deleted + decoded.blkCid == blkCid + decoded.proof == val.proof + decoded.isCell == true + decoded.cellCid == cellCid + + test "LeafMetadata cell variant backwards compatible (non-cell decodes correctly)": + let + val = LeafMetadata(deleted: false, blkCid: Cid.example, proof: nil) + decoded = LeafMetadata.decode(encode(val)).tryGet() + + check: + decoded.deleted == val.deleted + decoded.blkCid == val.blkCid + decoded.isCell == false diff --git a/tests/archivist/stores/repostore/testrepostore.nim b/tests/archivist/stores/repostore/testrepostore.nim new file mode 100644 index 00000000..0a5bd41d --- /dev/null +++ b/tests/archivist/stores/repostore/testrepostore.nim @@ -0,0 +1,1240 @@ +import std/strutils +import std/algorithm +import std/sequtils +import std/os +import std/tempfiles + +import pkg/questionable +import pkg/questionable/results + +import pkg/chronos +import pkg/stew/byteutils +import pkg/stew/bitseqs +import pkg/kvstore +import pkg/taskpools + +import pkg/archivist/stores +import pkg/archivist/stores/repostore/operations +import pkg/archivist/stores/repostore/types +import pkg/archivist/blocktype as bt +import pkg/archivist/clock +import pkg/archivist/merkletree +import pkg/archivist/merkletree/archivist +import pkg/archivist/utils + +import ../../../asynctest +import ../../helpers +import ../../helpers/mockclock +import ../../examples + +import ./overlays/helpers + +proc testRepoStore*( + name: string, + repoDsProvider: KVStoreProvider, + metaDsProvider: KVStoreProvider, + before: Before = nil, + after: After = nil, +) = + suite name: + var + repoDs: KVStore + metaDs: KVStore + mockClock: MockClock + repo: RepoStore + + let now: SecondsSince1970 = 123 + + setup: + if not before.isNil: + await before() + repoDs = repoDsProvider() + metaDs = metaDsProvider() + mockClock = MockClock.new() + mockClock.set(now) + repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 200'nb) + await repo.start() + + teardown: + await repo.close() + (await repoDs.close()).tryGet() + (await metaDs.close()).tryGet() + if not after.isNil: + await after() + + # ------------------------------------------------------- + # Quota / used-bytes tests + # ------------------------------------------------------- + + test "putBlock raises onBlockStored": + let newBlock1 = bt.Block.new("1".repeat(100).toBytes()).tryGet() + + var storedCid = Cid.example + proc onStored(cid: Cid): Future[void] {.async: (raises: []).} = + storedCid = cid + + repo.onBlockStored = onStored.some() + + discard (await putBlockWithOverlay(repo, newBlock1)).tryGet() + + check storedCid == newBlock1.cid + + test "Should update current used bytes on block put": + let blk = createTestBlock(200) + + check repo.quotaUsedBytes == 0'nb + discard (await putBlockWithOverlay(repo, blk)).tryGet + + check: + repo.quotaUsedBytes == 200'nb + + test "Should update current used bytes on block delete": + let blk = createTestBlock(100) + + check repo.quotaUsedBytes == 0'nb + let (treeCid, index) = (await putBlockWithOverlay(repo, blk)).tryGet + check repo.quotaUsedBytes == 100'nb + + (await repo.delBlock(treeCid, index)).tryGet + + check: + repo.quotaUsedBytes == 0'nb + + test "Should not update current used bytes if block exist": + let blk = createTestBlock(100) + + check repo.quotaUsedBytes == 0'nb + discard (await putBlockWithOverlay(repo, blk)).tryGet + check repo.quotaUsedBytes == 100'nb + + # put again + discard (await putBlockWithOverlay(repo, blk)).tryGet + check repo.quotaUsedBytes == 100'nb + + test "Should fail storing passed the quota": + let blk = createTestBlock(300) + + check repo.totalUsed == 0'nb + expect QuotaNotEnoughError: + discard (await putBlockWithOverlay(repo, blk)).tryGet + + test "Should reserve bytes": + let blk = createTestBlock(100) + + check repo.totalUsed == 0'nb + discard (await putBlockWithOverlay(repo, blk)).tryGet + check repo.totalUsed == 100'nb + + (await repo.reserve(100'nb)).tryGet + + check: + repo.totalUsed == 200'nb + repo.quotaUsedBytes == 100'nb + repo.quotaReservedBytes == 100'nb + + test "Should not reserve bytes over max quota": + let blk = createTestBlock(100) + + check repo.totalUsed == 0'nb + discard (await putBlockWithOverlay(repo, blk)).tryGet + check repo.totalUsed == 100'nb + + expect QuotaNotEnoughError: + (await repo.reserve(101'nb)).tryGet + + check: + repo.totalUsed == 100'nb + repo.quotaUsedBytes == 100'nb + repo.quotaReservedBytes == 0'nb + + test "Should release bytes": + discard createTestBlock(100) + + check repo.totalUsed == 0'nb + (await repo.reserve(100'nb)).tryGet + check repo.totalUsed == 100'nb + + (await repo.release(100'nb)).tryGet + + check: + repo.totalUsed == 0'nb + repo.quotaUsedBytes == 0'nb + repo.quotaReservedBytes == 0'nb + + test "Should not release bytes less than quota": + check repo.totalUsed == 0'nb + (await repo.reserve(100'nb)).tryGet + check repo.totalUsed == 100'nb + + expect QuotaNotEnoughError: + (await repo.release(101'nb)).tryGet + + check: + repo.totalUsed == 100'nb + repo.quotaUsedBytes == 0'nb + repo.quotaReservedBytes == 100'nb + + test "Should handle duplicate CIDs in same putBlocks batch correctly": + let blk = createTestBlock(100) + + let (_, tree) = makeManifestAndTree(@[blk, blk]).tryGet() + let treeCid = tree.rootCid.tryGet() + + var blocks = BitSeq.init(2) + + ( + await repo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = blocks + ) + ).tryGet() + + let + initialBytes = repo.quotaUsedBytes + initialBlocks = repo.totalBlocks + + proof0 = tree.getProof(0).tryGet() + proof1 = tree.getProof(1).tryGet() + + ( + await repo.putBlocks( + treeCid, @[(blk, 0.Natural, proof0), (blk, 1.Natural, proof1)] + ) + ).tryGet() + + check repo.quotaUsedBytes == initialBytes + 100.NBytes + check repo.totalBlocks == initialBlocks + 1 + + (await repo.delBlock(treeCid, 0)).tryGet() + check (await repo.getBlock(blk.cid)).isOk + + (await repo.delBlock(treeCid, 1)).tryGet() + check (await repo.getBlock(blk.cid)).isErr + + # ------------------------------------------------------- + # Empty block tests + # ------------------------------------------------------- + + test "Should put empty blocks": + let blk = Cid.example.emptyBlock.tryGet() + check (await putBlockWithOverlay(repo, blk)).isOk + + test "Should get empty blocks": + let blk = Cid.example.emptyBlock.tryGet() + + let got = await repo.getBlock(blk.cid) + check got.isOk + check got.get.cid == blk.cid + + test "Should delete empty blocks": + let blk = Cid.example.emptyBlock.tryGet() + check (await repo.delBlock(blk.cid)).isOk + + test "Should have empty block": + let blk = Cid.example.emptyBlock.tryGet() + + let has = await repo.hasBlock(blk.cid) + check has.isOk + check has.get + + test "fail getBlock": + let newBlock = bt.Block.new("New Kids on the Block".toBytes()).tryGet() + expect BlockNotFoundError: + discard (await repo.getBlock(newBlock.cid)).tryGet() + + test "fail hasBlock": + let newBlock = bt.Block.new("New Kids on the Block".toBytes()).tryGet() + check: + not (await repo.hasBlock(newBlock.cid)).tryGet() + not (await newBlock.cid in repo) + + # ------------------------------------------------------- + # Proof-based put/get/delete tests + # ------------------------------------------------------- + + test "Should put block with proof": + let + blk = createTestBlock(50) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var bits = BitSeq.init(1) + bits.setBit(0) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = bits)).tryGet() + (await repo.putBlock(treeCid, blk, 0.Natural, proof)).tryGet() + + check (await repo.hasBlock(blk.cid)).tryGet() + + test "Should get block with proof": + let + blk = createTestBlock(50) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var bits = BitSeq.init(1) + bits.setBit(0) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = bits)).tryGet() + (await repo.putBlock(treeCid, blk, 0.Natural, proof)).tryGet() + + let got = (await repo.getBlockAndProof(treeCid, 0.Natural)).tryGet() + check got[1].cid == blk.cid + check $got[2] == $proof + + test "Should delete block with proof": + let + blk = createTestBlock(50) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var bits = BitSeq.init(1) + bits.setBit(0) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = bits)).tryGet() + (await repo.putBlock(treeCid, blk, 0.Natural, proof)).tryGet() + check (await repo.hasBlock(blk.cid)).tryGet() + + (await repo.delBlock(treeCid, 0.Natural)).tryGet() + check not (await repo.hasBlock(blk.cid)).tryGet() + + test "Should get blocks with proofs": + let + dataset = (await makeRandomBlocks(datasetSize = 768, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + # Use a large enough quota + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(3) + bits.setBit(0) + bits.setBit(1) + bits.setBit(2) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + let + proof0 = tree.getProof(0).tryGet() + proof1 = tree.getProof(1).tryGet() + proof2 = tree.getProof(2).tryGet() + + ( + await bigRepo.putBlocks( + treeCid, + @[ + (dataset[0], 0.Natural, proof0), + (dataset[1], 1.Natural, proof1), + (dataset[2], 2.Natural, proof2), + ], + ) + ).tryGet() + + let unsorted = ( + await bigRepo.getBlocksAndProofs(treeCid, @[0.Natural, 1.Natural, 2.Natural]) + ).tryGet() + let results = unsorted.sortedByIt(it[0]) + + check results.len == 3 + check results[0][1].cid == dataset[0].cid + check $results[0][2] == $proof0 + check results[1][1].cid == dataset[1].cid + check $results[1][2] == $proof1 + check results[2][1].cid == dataset[2].cid + check $results[2][2] == $proof2 + + test "Should handle non-existent block with proof": + let + blk = createTestBlock(50) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + + # No overlay or block stored - getBlockAndProof should fail + let got = await repo.getBlockAndProof(treeCid, 0.Natural) + check got.isErr + + test "Should delete blocks with proofs": + let + dataset = (await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(2) + bits.setBit(0) + bits.setBit(1) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + let + proof0 = tree.getProof(0).tryGet() + proof1 = tree.getProof(1).tryGet() + + ( + await bigRepo.putBlocks( + treeCid, @[(dataset[0], 0.Natural, proof0), (dataset[1], 1.Natural, proof1)] + ) + ).tryGet() + + (await bigRepo.delBlocks(treeCid, @[0.Natural, 1.Natural])).tryGet() + + check not (await bigRepo.hasBlock(dataset[0].cid)).tryGet() + check not (await bigRepo.hasBlock(dataset[1].cid)).tryGet() + + test "Should handle partial failures in delete blocks": + let + dataset = (await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(2) + bits.setBit(0) + # index 1 not set - block at index 1 was never stored + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + let proof0 = tree.getProof(0).tryGet() + + (await bigRepo.putBlocks(treeCid, @[(dataset[0], 0.Natural, proof0)])).tryGet() + + # Deleting index 1 (never stored) should still succeed + (await bigRepo.delBlocks(treeCid, @[0.Natural, 1.Natural])).tryGet() + + check not (await bigRepo.hasBlock(dataset[0].cid)).tryGet() + + # ------------------------------------------------------- + # hasBlocks tests + # ------------------------------------------------------- + + test "Should handle has blocks": + let + dataset = (await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(2) + bits.setBit(0) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + let proof0 = tree.getProof(0).tryGet() + + (await bigRepo.putBlocks(treeCid, @[(dataset[0], 0.Natural, proof0)])).tryGet() + + let unsorted = + (await bigRepo.hasBlocks(treeCid, @[0.Natural, 1.Natural])).tryGet() + let results = unsorted.sortedByIt(it[0]) + + check results.len == 2 + check results[0][0] == 0.Natural + check results[0][1] == true + check results[1][0] == 1.Natural + check results[1][1] == false + + # ------------------------------------------------------- + # Batch put tests + # ------------------------------------------------------- + + test "Should handle batch put blocks": + let + dataset = (await makeRandomBlocks(datasetSize = 768, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(3) + bits.setBit(0) + bits.setBit(1) + bits.setBit(2) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + let + proof0 = tree.getProof(0).tryGet() + proof1 = tree.getProof(1).tryGet() + proof2 = tree.getProof(2).tryGet() + + ( + await bigRepo.putBlocks( + treeCid, + @[ + (dataset[0], 0.Natural, proof0), + (dataset[1], 1.Natural, proof1), + (dataset[2], 2.Natural, proof2), + ], + ) + ).tryGet() + + check (await bigRepo.hasBlock(dataset[0].cid)).tryGet() + check (await bigRepo.hasBlock(dataset[1].cid)).tryGet() + check (await bigRepo.hasBlock(dataset[2].cid)).tryGet() + + test "Should handle batch get blocks": + let + dataset = (await makeRandomBlocks(datasetSize = 768, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(3) + bits.setBit(0) + bits.setBit(1) + bits.setBit(2) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + let + proof0 = tree.getProof(0).tryGet() + proof1 = tree.getProof(1).tryGet() + proof2 = tree.getProof(2).tryGet() + + ( + await bigRepo.putBlocks( + treeCid, + @[ + (dataset[0], 0.Natural, proof0), + (dataset[1], 1.Natural, proof1), + (dataset[2], 2.Natural, proof2), + ], + ) + ).tryGet() + + let unsorted = + (await bigRepo.getBlocks(treeCid, @[0.Natural, 1.Natural, 2.Natural])).tryGet() + let results = unsorted.sortedByIt(it[0]) + + check results.len == 3 + check results[0][1].cid == dataset[0].cid + check results[1][1].cid == dataset[1].cid + check results[2][1].cid == dataset[2].cid + + test "Should handle concurrent batch operations": + let + dataset = (await makeRandomBlocks(datasetSize = 768, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(3) + bits.setBit(0) + bits.setBit(1) + bits.setBit(2) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + let + proof0 = tree.getProof(0).tryGet() + proof1 = tree.getProof(1).tryGet() + proof2 = tree.getProof(2).tryGet() + + # Store blocks concurrently + let putFuts = await allFinished( + @[ + bigRepo.putBlock(treeCid, dataset[0], 0.Natural, proof0), + bigRepo.putBlock(treeCid, dataset[1], 1.Natural, proof1), + bigRepo.putBlock(treeCid, dataset[2], 2.Natural, proof2), + ] + ) + + for f in putFuts: + check not f.failed + check f.read.isOk + + # Get blocks concurrently + let getFuts = await allFinished( + @[ + bigRepo.getBlock(treeCid, 0.Natural), + bigRepo.getBlock(treeCid, 1.Natural), + bigRepo.getBlock(treeCid, 2.Natural), + ] + ) + + for f in getFuts: + check not f.failed + check f.read.isOk + + test "Should handle empty batch operations": + let + dataset = (await makeRandomBlocks(datasetSize = 256, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(1) + bits.setBit(0) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + # Empty putBlocks + (await bigRepo.putBlocks(treeCid, newSeq[(bt.Block, Natural, ArchivistProof)]())).tryGet() + + # Empty getBlocks + let emptyGet = (await bigRepo.getBlocks(treeCid, newSeq[Natural]())).tryGet() + check emptyGet.len == 0 + + # Empty delBlocks + (await bigRepo.delBlocks(treeCid, newSeq[Natural]())).tryGet() + + # Empty hasBlocks + let emptyHas = (await bigRepo.hasBlocks(treeCid, newSeq[Natural]())).tryGet() + check emptyHas.len == 0 + + test "Should handle single item batch": + let + blk = createTestBlock(50) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var bits = BitSeq.init(1) + bits.setBit(0) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = bits)).tryGet() + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() + + let got = (await repo.getBlocks(treeCid, @[0.Natural])).tryGet() + check got.len == 1 + check got[0][1].cid == blk.cid + + test "Should handle batch with duplicate CIDs": + let + blk = createTestBlock(50) + (_, tree) = makeManifestAndTree(@[blk, blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof0 = tree.getProof(0).tryGet() + proof1 = tree.getProof(1).tryGet() + + var bits = BitSeq.init(2) + bits.setBit(0) + bits.setBit(1) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = bits)).tryGet() + + ( + await repo.putBlocks( + treeCid, @[(blk, 0.Natural, proof0), (blk, 1.Natural, proof1)] + ) + ).tryGet() + + # Block stored once despite appearing at two indices + check (await repo.hasBlock(blk.cid)).tryGet() + check repo.totalBlocks == 1 + + test "Should handle batch with missing blocks": + let + blk = createTestBlock(50) + (_, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var bits = BitSeq.init(1) + bits.setBit(0) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = bits)).tryGet() + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() + + # Request index 0 (exists) and index 5 (missing) + let results = (await repo.getBlocks(treeCid, @[0.Natural, 5.Natural])).tryGet() + + # Only index 0 should be returned + check results.len == 1 + check results[0][0] == 0.Natural + + # ------------------------------------------------------- + # Batch delete tests + # ------------------------------------------------------- + + test "Should handle batch delete with missing blocks": + let + dataset = (await makeRandomBlocks(datasetSize = 256, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(1) + bits.setBit(0) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + let proof = tree.getProof(0).tryGet() + (await bigRepo.putBlocks(treeCid, @[(dataset[0], 0.Natural, proof)])).tryGet() + + # Delete index 0 (exists) and index 5 (does not exist) + (await bigRepo.delBlocks(treeCid, @[0.Natural, 5.Natural])).tryGet() + + check not (await bigRepo.hasBlock(dataset[0].cid)).tryGet() + + test "Should handle batch delete with partial success": + let + dataset = (await makeRandomBlocks(datasetSize = 768, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(3) + bits.setBit(0) + bits.setBit(1) + bits.setBit(2) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + let + proof0 = tree.getProof(0).tryGet() + proof2 = tree.getProof(2).tryGet() + + # Only store indices 0 and 2; index 1 is not stored + + ( + await bigRepo.putBlocks( + treeCid, @[(dataset[0], 0.Natural, proof0), (dataset[2], 2.Natural, proof2)] + ) + ).tryGet() + + # Delete all three - index 1 was never stored but delete should still succeed + (await bigRepo.delBlocks(treeCid, @[0.Natural, 1.Natural, 2.Natural])).tryGet() + + check not (await bigRepo.hasBlock(dataset[0].cid)).tryGet() + check not (await bigRepo.hasBlock(dataset[2].cid)).tryGet() + + test "Should handle batch delete with all missing": + let + dataset = (await makeRandomBlocks(datasetSize = 256, blockSize = 256'nb)).tryGet + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits = BitSeq.init(1) + bits.setBit(0) + + ( + await bigRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits + ) + ).tryGet() + + # Delete indices that were never stored - should succeed without error + (await bigRepo.delBlocks(treeCid, @[5.Natural, 10.Natural, 99.Natural])).tryGet() + + test "Should handle batch delete with shared CIDs": + # Two overlays sharing the same block - deleting from one should not remove the block + let + pool = (await makeRandomBlocks(datasetSize = 768, blockSize = 256'nb)).tryGet + sharedBlk = pool[0] + unique1 = pool[1] + unique2 = pool[2] + + (_, tree1) = makeManifestAndTree(@[sharedBlk, unique1]).tryGet() + treeCid1 = tree1.rootCid.tryGet() + (_, tree2) = makeManifestAndTree(@[sharedBlk, unique2]).tryGet() + treeCid2 = tree2.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits1 = BitSeq.init(2) + bits1.setBit(0) + var bits2 = BitSeq.init(2) + bits2.setBit(0) + + ( + await bigRepo.putOverlay( + treeCid = treeCid1, status = Completed.some, blocks = bits1 + ) + ).tryGet() + + ( + await bigRepo.putOverlay( + treeCid = treeCid2, status = Completed.some, blocks = bits2 + ) + ).tryGet() + + let + proof1 = tree1.getProof(0).tryGet() + proof2 = tree2.getProof(0).tryGet() + + (await bigRepo.putBlocks(treeCid1, @[(sharedBlk, 0.Natural, proof1)])).tryGet() + (await bigRepo.putBlocks(treeCid2, @[(sharedBlk, 0.Natural, proof2)])).tryGet() + + check (await bigRepo.blockRefCount(sharedBlk.cid)).tryGet() == 2.Natural + + # Delete from tree1 only - block should remain (refCount 2 -> 1) + (await bigRepo.delBlocks(treeCid1, @[0.Natural])).tryGet() + check (await bigRepo.hasBlock(sharedBlk.cid)).tryGet() + check (await bigRepo.blockRefCount(sharedBlk.cid)).tryGet() == 1.Natural + + test "Should handle batch delete with shared CIDs partial": + let + dataset = (await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb)).tryGet + sharedBlk = dataset[0] + uniqueBlk = dataset[1] + + (_, tree) = makeManifestAndTree(@[sharedBlk, uniqueBlk]).tryGet() + treeCid1 = tree.rootCid.tryGet() + + (_, tree2) = makeManifestAndTree(@[sharedBlk]).tryGet() + treeCid2 = tree2.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits1 = BitSeq.init(2) + bits1.setBit(0) + bits1.setBit(1) + + var bits2 = BitSeq.init(1) + bits2.setBit(0) + + ( + await bigRepo.putOverlay( + treeCid = treeCid1, status = Completed.some, blocks = bits1 + ) + ).tryGet() + + ( + await bigRepo.putOverlay( + treeCid = treeCid2, status = Completed.some, blocks = bits2 + ) + ).tryGet() + + let + tproof0 = tree.getProof(0).tryGet() + tproof1 = tree.getProof(1).tryGet() + t2proof0 = tree2.getProof(0).tryGet() + + ( + await bigRepo.putBlocks( + treeCid1, @[(sharedBlk, 0.Natural, tproof0), (uniqueBlk, 1.Natural, tproof1)] + ) + ).tryGet() + + (await bigRepo.putBlocks(treeCid2, @[(sharedBlk, 0.Natural, t2proof0)])).tryGet() + + # sharedBlk has refCount 2, uniqueBlk has refCount 1 + check (await bigRepo.blockRefCount(sharedBlk.cid)).tryGet() == 2.Natural + check (await bigRepo.blockRefCount(uniqueBlk.cid)).tryGet() == 1.Natural + + # Delete all blocks from treeCid1 + (await bigRepo.delBlocks(treeCid1, @[0.Natural, 1.Natural])).tryGet() + + # uniqueBlk should be gone, sharedBlk should remain (still in treeCid2) + check not (await bigRepo.hasBlock(uniqueBlk.cid)).tryGet() + check (await bigRepo.hasBlock(sharedBlk.cid)).tryGet() + + test "Should handle batch delete with shared CIDs all shared": + let + pool = (await makeRandomBlocks(datasetSize = 768, blockSize = 256'nb)).tryGet + sharedBlk = pool[0] + unique1 = pool[1] + unique2 = pool[2] + + (_, tree1) = makeManifestAndTree(@[sharedBlk, unique1]).tryGet() + treeCid1 = tree1.rootCid.tryGet() + (_, tree2) = makeManifestAndTree(@[sharedBlk, unique2]).tryGet() + treeCid2 = tree2.rootCid.tryGet() + + let bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + + var bits1 = BitSeq.init(2) + bits1.setBit(0) + var bits2 = BitSeq.init(2) + bits2.setBit(0) + + ( + await bigRepo.putOverlay( + treeCid = treeCid1, status = Completed.some, blocks = bits1 + ) + ).tryGet() + + ( + await bigRepo.putOverlay( + treeCid = treeCid2, status = Completed.some, blocks = bits2 + ) + ).tryGet() + + let + proof1 = tree1.getProof(0).tryGet() + proof2 = tree2.getProof(0).tryGet() + + (await bigRepo.putBlocks(treeCid1, @[(sharedBlk, 0.Natural, proof1)])).tryGet() + (await bigRepo.putBlocks(treeCid2, @[(sharedBlk, 0.Natural, proof2)])).tryGet() + + check (await bigRepo.blockRefCount(sharedBlk.cid)).tryGet() == 2.Natural + + # Delete from both trees - block should be removed after second delete + (await bigRepo.delBlocks(treeCid1, @[0.Natural])).tryGet() + check (await bigRepo.hasBlock(sharedBlk.cid)).tryGet() + + (await bigRepo.delBlocks(treeCid2, @[0.Natural])).tryGet() + check not (await bigRepo.hasBlock(sharedBlk.cid)).tryGet() + + test "Should not allow non-orphan blocks to be deleted directly": + let + innerRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 1000'nb) + dataset = (await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb)).tryGet + blk = dataset[0] + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var blocks = BitSeq.init(1) + blocks.setBit(0) + + ( + await innerRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = blocks + ) + ).tryGet + (await innerRepo.putBlock(treeCid, blk, 0, proof)).tryGet + let err = (await innerRepo.delBlock(blk.cid)).error() + check err.msg == + "Directly deleting a block that is part of a dataset is not allowed." + + test "Should allow non-orphan blocks to be deleted by dataset reference": + let + innerRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 1000'nb) + dataset = (await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb)).tryGet + blk = dataset[0] + (_, tree) = makeManifestAndTree(dataset).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var blocks = BitSeq.init(1) + blocks.setBit(0) + + ( + await innerRepo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = blocks + ) + ).tryGet + (await innerRepo.putBlock(treeCid, blk, 0, proof)).tryGet + + (await innerRepo.delBlock(treeCid, 0.Natural)).tryGet() + check not (await blk.cid in innerRepo) + + test "Should not delete a non-orphan block until it is deleted from all parent datasets": + let + innerRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 1024'nb) + blockPool = + (await makeRandomBlocks(datasetSize = 768, blockSize = 256'nb)).tryGet + + dataset1 = @[blockPool[0], blockPool[1]] + dataset2 = @[blockPool[1], blockPool[2]] + sharedBlock = blockPool[1] + + (_, tree1) = makeManifestAndTree(dataset1).tryGet() + treeCid1 = tree1.rootCid.tryGet() + (_, tree2) = makeManifestAndTree(dataset2).tryGet() + treeCid2 = tree2.rootCid.tryGet() + + # Create overlay for tree1 + var blocks1 = BitSeq.init(2) + blocks1.setBit(0) + blocks1.setBit(1) + + ( + await innerRepo.putOverlay( + treeCid = treeCid1, status = Completed.some, blocks = blocks1 + ) + ).tryGet() + + # Put dataset1 + let + proof1_0 = tree1.getProof(0).tryGet() + proof1_1 = tree1.getProof(1).tryGet() + + ( + await innerRepo.putBlocks( + treeCid1, + @[(blockPool[0], 0.Natural, proof1_0), (sharedBlock, 1.Natural, proof1_1)], + ) + ).tryGet() + + # Shared block should exist with refCount = 1 + check (await innerRepo.blockRefCount(sharedBlock.cid)).tryGet() == 1.Natural + + # Create overlay for tree2 + var blocks2 = BitSeq.init(2) + blocks2.setBit(0) + blocks2.setBit(1) + + ( + await innerRepo.putOverlay( + treeCid = treeCid2, status = Completed.some, blocks = blocks2 + ) + ).tryGet() + + # Put dataset2 + let + proof2_0 = tree2.getProof(0).tryGet() + proof2_1 = tree2.getProof(1).tryGet() + + ( + await innerRepo.putBlocks( + treeCid2, + @[(sharedBlock, 0.Natural, proof2_0), (blockPool[2], 1.Natural, proof2_1)], + ) + ).tryGet() + + # Shared block should now have refCount = 2 + check (await innerRepo.blockRefCount(sharedBlock.cid)).tryGet() == 2.Natural + + # Delete from tree1 + (await innerRepo.delBlock(treeCid1, 1.Natural)).tryGet() + check (await innerRepo.blockRefCount(sharedBlock.cid)).tryGet() == 1.Natural + check (await sharedBlock.cid in innerRepo) + + # Delete from tree2 + (await innerRepo.delBlock(treeCid2, 0.Natural)).tryGet() + check not (await sharedBlock.cid in innerRepo) + + # ------------------------------------------------------- + # Batch by CID tests (getBlocks by seq[Cid] overload) + # ------------------------------------------------------- + + test "should get multiple blocks by CID": + let + cidRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 2000'nb) + blk1 = createTestBlock(100) + blk2 = createTestBlock(150) + blk3 = createTestBlock(200) + + # Store blocks via overlay API + discard (await putBlockWithOverlay(cidRepo, blk1)).tryGet() + discard (await putBlockWithOverlay(cidRepo, blk2)).tryGet() + discard (await putBlockWithOverlay(cidRepo, blk3)).tryGet() + + # Retrieve all three blocks + let blocks = (await cidRepo.getBlocks(@[blk1.cid, blk2.cid, blk3.cid])).tryGet() + let returnedCids = blocks.mapIt(it.cid) + + check blocks.len == 3 + check blk1.cid in returnedCids + check blk2.cid in returnedCids + check blk3.cid in returnedCids + + test "should return empty seq for empty input": + let blocks = (await repo.getBlocks(newSeq[Cid]())).tryGet() + check blocks.len == 0 + + test "should skip missing CIDs": + let + cidRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 2000'nb) + blk1 = createTestBlock(100) + blk2 = createTestBlock(150) + + # Store only 2 blocks + discard (await putBlockWithOverlay(cidRepo, blk1)).tryGet() + discard (await putBlockWithOverlay(cidRepo, blk2)).tryGet() + + # Request 3 CIDs (one missing) + let missingCid = Cid.example + let blocks = (await cidRepo.getBlocks(@[blk1.cid, missingCid, blk2.cid])).tryGet() + + # Should return only the 2 existing blocks + check blocks.len == 2 + + test "should handle empty CIDs": + let + blk1 = createTestBlock(100) + emptyBlk = Cid.example.emptyBlock.tryGet() + + # Store one real block + discard (await putBlockWithOverlay(repo, blk1)).tryGet() + + # Request real + empty CID + let blocks = (await repo.getBlocks(@[blk1.cid, emptyBlk.cid])).tryGet() + + # Should return the real block and synthesized empty block + check blocks.len == 2 + + # ------------------------------------------------------- + # listBlocks tests + # ------------------------------------------------------- + + test "listBlocks Blocks": + let + bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + newBlock1 = bt.Block.new("1".repeat(100).toBytes()).tryGet() + newBlock2 = bt.Block.new("2".repeat(100).toBytes()).tryGet() + newBlock3 = bt.Block.new("3".repeat(100).toBytes()).tryGet() + blocks = @[newBlock1, newBlock2, newBlock3] + putHandles = await allFinished(blocks.mapIt(putBlockWithOverlay(bigRepo, it))) + + for handle in putHandles: + check not handle.failed + check handle.read.isOk + + let cidsIter = (await bigRepo.listBlocks(blockType = BlockType.Block)).tryGet() + + var count = 0 + for c in cidsIter: + if cid =? await c: + check (await bigRepo.hasBlock(cid)).tryGet() + count.inc + + check count == 3 + + test "listBlocks Manifest": + let + bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + newBlock = bt.Block.new("New Kids on the Block".toBytes()).tryGet() + newBlock1 = bt.Block.new("1".repeat(100).toBytes()).tryGet() + newBlock2 = bt.Block.new("2".repeat(100).toBytes()).tryGet() + newBlock3 = bt.Block.new("3".repeat(100).toBytes()).tryGet() + (manifest, tree) = + makeManifestAndTree(@[newBlock, newBlock1, newBlock2, newBlock3]).tryGet() + blocks = @[newBlock1, newBlock2, newBlock3] + manifestBlock = + Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() + treeBlock = Block.new(tree.encode()).tryGet() + putHandles = await allFinished( + (@[treeBlock, manifestBlock] & blocks).mapIt(bigRepo.putBlock(it)) + ) + + for handle in putHandles: + check not handle.failed + check handle.read.isOk + + let cidsIter = (await bigRepo.listBlocks(blockType = BlockType.Manifest)).tryGet() + + var count = 0 + for c in cidsIter: + if cid =? await c: + check manifestBlock.cid == cid + check (await bigRepo.hasBlock(cid)).tryGet() + count.inc + + check count == 1 + + test "listBlocks Both": + let + bigRepo = + RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 5000'nb) + newBlock = bt.Block.new("New Kids on the Block".toBytes()).tryGet() + newBlock1 = bt.Block.new("1".repeat(100).toBytes()).tryGet() + newBlock2 = bt.Block.new("2".repeat(100).toBytes()).tryGet() + newBlock3 = bt.Block.new("3".repeat(100).toBytes()).tryGet() + (manifest, tree) = + makeManifestAndTree(@[newBlock, newBlock1, newBlock2, newBlock3]).tryGet() + blocks = @[newBlock1, newBlock2, newBlock3] + manifestBlock = + Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() + treeBlock = Block.new(tree.encode()).tryGet() + + # Store manifest and tree blocks directly (not overlay API) + let manifestHandles = + await allFinished((@[treeBlock, manifestBlock]).mapIt(bigRepo.putBlock(it))) + + for handle in manifestHandles: + check not handle.failed + check handle.read.isOk + + # Store data blocks via overlay API + let dataHandles = + await allFinished(blocks.mapIt(putBlockWithOverlay(bigRepo, it))) + + for handle in dataHandles: + check not handle.failed + check handle.read.isOk + + let cidsIter = (await bigRepo.listBlocks(blockType = BlockType.Both)).tryGet() + + var count = 0 + for c in cidsIter: + if cid =? await c: + check (await bigRepo.hasBlock(cid)).tryGet() + count.inc + + check count == 5 + +proc runFsSqliteTests() = + let repoDir = createTempDir("archivist-", "-repostore") + + testRepoStore( + "RepoStore FS+SQLite backend", + repoDsProvider = proc(): KVStore = + if not dirExists(repoDir): + createDir(repoDir) + let tp = Taskpool.new() + FSKVStore.new(repoDir, tp, depth = 5).tryGet(), + metaDsProvider = proc(): KVStore = + let tp = Taskpool.new() + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + after = proc(): Future[void] {.async.} = + os.removeDir(repoDir), + ) + +runFsSqliteTests() diff --git a/tests/archivist/stores/repostore/teststartup.nim b/tests/archivist/stores/repostore/teststartup.nim new file mode 100644 index 00000000..a932df24 --- /dev/null +++ b/tests/archivist/stores/repostore/teststartup.nim @@ -0,0 +1,83 @@ +import std/os +import std/tempfiles + +import pkg/questionable/results + +import pkg/chronos +import pkg/kvstore +import pkg/kvstore/fsds +import pkg/taskpools + +import pkg/archivist/stores +import pkg/archivist/stores/repostore/types + +import ../../../asynctest +import ../../helpers + +import ./overlays/helpers + +proc testStartup*( + name: string, + repoDsProvider: KVStoreProvider, + metaDsProvider: KVStoreProvider, + before: Before = nil, + after: After = nil, +) = + suite name: + var + repoDs: KVStore + metaDs: KVStore + + setup: + if not isNil(before): + await before() + repoDs = repoDsProvider() + metaDs = metaDsProvider() + + teardown: + (await repoDs.close()).tryGet + (await metaDs.close()).tryGet + if not isNil(after): + await after() + + test "Should set started flag once started": + let repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = 200'nb) + await repo.start() + check repo.started + + test "Should set started flag to false once stopped": + let repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = 200'nb) + await repo.start() + await repo.stop() + check not repo.started + + test "Should allow start to be called multiple times": + let repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = 200'nb) + await repo.start() + await repo.start() + check repo.started + + test "Should allow stop to be called multiple times": + let repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = 200'nb) + await repo.stop() + await repo.stop() + check not repo.started + +proc runFsSqliteTests() = + let repoDir = createTempDir("archivist-", "-repostore") + + testStartup( + "RepoStore startup FS+SQLite backend", + repoDsProvider = proc(): KVStore = + if not dirExists(repoDir): + createDir(repoDir) + let tp = Taskpool.new() + FSKVStore.new(repoDir, tp, depth = 5).tryGet(), + metaDsProvider = proc(): KVStore = + let tp = Taskpool.new() + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + after = proc(): Future[void] {.async.} = + os.removeDir(repoDir), + ) + +runFsSqliteTests() diff --git a/tests/archivist/stores/testcachestore.nim b/tests/archivist/stores/testcachestore.nim deleted file mode 100644 index f27470f0..00000000 --- a/tests/archivist/stores/testcachestore.nim +++ /dev/null @@ -1,70 +0,0 @@ -import std/strutils - -import pkg/chronos -import pkg/stew/byteutils -import pkg/questionable/results -import pkg/archivist/stores/cachestore -import pkg/archivist/chunker - -import ./commonstoretests - -import ../../asynctest -import ../helpers - -suite "Cache Store": - var - newBlock, newBlock1, newBlock2, newBlock3: Block - store: CacheStore - - setup: - newBlock = Block.new("New Kids on the Block".toBytes()).tryGet() - newBlock1 = Block.new("1".repeat(100).toBytes()).tryGet() - newBlock2 = Block.new("2".repeat(100).toBytes()).tryGet() - newBlock3 = Block.new("3".repeat(100).toBytes()).tryGet() - store = CacheStore.new() - - test "constructor": - # cache size cannot be smaller than chunk size - expect ValueError: - discard CacheStore.new(cacheSize = 1, chunkSize = 2) - - store = CacheStore.new(cacheSize = 100, chunkSize = 1) - check store.currentSize == 0'nb - - store = CacheStore.new(@[newBlock1, newBlock2, newBlock3]) - check store.currentSize == 300'nb - - # initial cache blocks total more than cache size, currentSize should - # never exceed max cache size - store = CacheStore.new( - blocks = @[newBlock1, newBlock2, newBlock3], cacheSize = 200, chunkSize = 1 - ) - check store.currentSize == 200'nb - - # cache size cannot be less than chunks size - expect ValueError: - discard CacheStore.new(cacheSize = 99, chunkSize = 100) - - test "putBlock": - (await store.putBlock(newBlock1)).tryGet() - check (await store.hasBlock(newBlock1.cid)).tryGet() - - # block size bigger than entire cache - store = CacheStore.new(cacheSize = 99, chunkSize = 98) - (await store.putBlock(newBlock1)).tryGet() - check not (await store.hasBlock(newBlock1.cid)).tryGet() - - # block being added causes removal of LRU block - store = - CacheStore.new(@[newBlock1, newBlock2, newBlock3], cacheSize = 200, chunkSize = 1) - check: - not (await store.hasBlock(newBlock1.cid)).tryGet() - (await store.hasBlock(newBlock2.cid)).tryGet() - (await store.hasBlock(newBlock2.cid)).tryGet() - store.currentSize.int == newBlock2.data.len + newBlock3.data.len # 200 - -commonBlockStoreTests( - "Cache", - proc(): BlockStore = - BlockStore(CacheStore.new(cacheSize = 1000, chunkSize = 1)), -) diff --git a/tests/archivist/stores/testkeyutils.nim b/tests/archivist/stores/testkeyutils.nim index 5d1f0f8c..1b98ba0a 100644 --- a/tests/archivist/stores/testkeyutils.nim +++ b/tests/archivist/stores/testkeyutils.nim @@ -18,7 +18,6 @@ import pkg/archivist/clock import ../../asynctest import ../helpers/mocktimer -import ../helpers/mockrepostore import ../helpers/mockclock import ../examples @@ -70,24 +69,24 @@ suite "KeyUtils": namespaces[2].value == expectedPrefix namespaces[3].value == expectedPostfix - test "createBlockExpirationMetadataKey should create block TTL key": + test "blockMetaKey should create block metadata key": let cid = Cid.example - let key = !createBlockExpirationMetadataKey(cid).option + let key = !blockMetaKey(cid).option let namespaces = key.namespaces check: namespaces.len == 3 namespaces[0].value == ArchivistMetaNamespace - namespaces[1].value == "ttl" + namespaces[1].value == "blocks" namespaces[2].value == $cid - test "createBlockExpirationMetadataQueryKey should create key for all block TTL entries": - let key = !createBlockExpirationMetadataQueryKey().option + test "blockMetaKeyQuery should create key for all block metadata entries": + let key = !blockMetaKeyQuery().option let namespaces = key.namespaces check: namespaces.len == 3 namespaces[0].value == ArchivistMetaNamespace - namespaces[1].value == "ttl" + namespaces[1].value == "blocks" namespaces[2].value == "*" diff --git a/tests/archivist/stores/testmaintenance.nim b/tests/archivist/stores/testmaintenance.nim index 0a2c3adb..c48f02f0 100644 --- a/tests/archivist/stores/testmaintenance.nim +++ b/tests/archivist/stores/testmaintenance.nim @@ -7,172 +7,186 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +import std/sets + import pkg/chronos +import pkg/kvstore +import pkg/taskpools +import pkg/questionable +import pkg/questionable/results +import pkg/stew/byteutils + +import pkg/libp2p/multicodec + import pkg/archivist/blocktype as bt -import pkg/archivist/stores/repostore -import pkg/archivist/clock +import pkg/archivist/stores import ../../asynctest -import ../helpers import ../helpers/mocktimer -import ../helpers/mockrepostore import ../helpers/mockclock import ../examples import archivist/stores/maintenance suite "BlockMaintainer": - var mockRepoStore: MockRepoStore - var interval: Duration - var mockTimer: MockTimer - var mockClock: MockClock - - var blockMaintainer: BlockMaintainer - - var testBe1: BlockExpiration - var testBe2: BlockExpiration - var testBe3: BlockExpiration - - proc createTestExpiration(expiry: SecondsSince1970): BlockExpiration = - BlockExpiration(cid: bt.Block.example.cid, expiry: expiry) + var + tp: Taskpool + repoDs: KVStore + metaDs: KVStore + mockClock: MockClock + mockTimer: MockTimer + repo: RepoStore + maintainer: BlockMaintainer + interval: Duration setup: + tp = Taskpool.new() + repoDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() mockClock = MockClock.new() mockClock.set(100) - - testBe1 = createTestExpiration(200) - testBe2 = createTestExpiration(300) - testBe3 = createTestExpiration(400) - - mockRepoStore = MockRepoStore.new() - mockRepoStore.testBlockExpirations.add(testBe1) - mockRepoStore.testBlockExpirations.add(testBe2) - mockRepoStore.testBlockExpirations.add(testBe3) - + repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 2000'nb) interval = 1.days mockTimer = MockTimer.new() + maintainer = + BlockMaintainer.new(repo, interval, timer = mockTimer, clock = mockClock) - blockMaintainer = BlockMaintainer.new( - mockRepoStore, interval, numberOfBlocksPerInterval = 2, mockTimer, mockClock - ) + teardown: + (await repoDs.close()).tryGet() + (await metaDs.close()).tryGet() + tp.shutdown() test "Start should start timer at provided interval": - blockMaintainer.start() + maintainer.start() check mockTimer.startCalled == 1 check mockTimer.mockInterval == interval test "Stop should stop timer": - await blockMaintainer.stop() + await maintainer.stop() check mockTimer.stopCalled == 1 - test "Timer callback should call getBlockExpirations on RepoStore": - blockMaintainer.start() + test "Should not drop overlays that have not expired": + let treeCid = Cid.example + + (await repo.putOverlay(treeCid, status = Completed.some, expiry = 200)).tryGet() + + maintainer.start() await mockTimer.invokeCallback() - check: - mockRepoStore.getBeMaxNumber == 2 - mockRepoStore.getBeOffset == 0 + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.expiry == 200 - test "Subsequent timer callback should call getBlockExpirations on RepoStore with offset": - blockMaintainer.start() + test "Should drop overlay that has expired": + let treeCid = Cid.example + + (await repo.putOverlay(treeCid, status = Completed.some, expiry = 50)).tryGet() + + maintainer.start() await mockTimer.invokeCallback() + + let res = await repo.getOverlay(treeCid) + check res.isErr + + test "Should drop only expired overlays": + let + cid1 = Cid + .init( + CIDv1, + multiCodec("codex-manifest"), + bt.Block.new("a".toBytes).tryGet().cid.mhash.tryGet(), + ) + .tryGet() + cid2 = Cid + .init( + CIDv1, + multiCodec("codex-manifest"), + bt.Block.new("b".toBytes).tryGet().cid.mhash.tryGet(), + ) + .tryGet() + cid3 = Cid + .init( + CIDv1, + multiCodec("codex-manifest"), + bt.Block.new("c".toBytes).tryGet().cid.mhash.tryGet(), + ) + .tryGet() + + (await repo.putOverlay(cid1, status = Completed.some, expiry = 50)).tryGet() + (await repo.putOverlay(cid2, status = Completed.some, expiry = 200)).tryGet() + (await repo.putOverlay(cid3, status = Completed.some, expiry = 90)).tryGet() + + maintainer.start() await mockTimer.invokeCallback() - check: - mockRepoStore.getBeMaxNumber == 2 - mockRepoStore.getBeOffset == 2 + # cid1 (expiry=50) and cid3 (expiry=90) expired, cid2 (expiry=200) retained + check (await repo.getOverlay(cid1)).isErr + check (await repo.getOverlay(cid2)).isOk + check (await repo.getOverlay(cid3)).isErr - test "Timer callback should delete no blocks if none are expired": - blockMaintainer.start() - await mockTimer.invokeCallback() + test "Should not drop overlays with default TTL that have not expired": + let treeCid = Cid.example + # ZeroSeconds triggers default TTL: now() + overlayTtl - check: - mockRepoStore.delBlockCids.len == 0 + (await repo.putOverlay(treeCid, status = Completed.some, expiry = 0)).tryGet() - test "Timer callback should delete one block if it is expired": - mockClock.set(250) - blockMaintainer.start() + # Clock is still at 100, well before the default TTL expiry + maintainer.start() await mockTimer.invokeCallback() - check: - mockRepoStore.delBlockCids == [testBe1.cid] + let meta = (await repo.getOverlay(treeCid)).tryGet() + check meta.expiry > 0 - test "Timer callback should delete multiple blocks if they are expired": - mockClock.set(500) - blockMaintainer.start() - await mockTimer.invokeCallback() + test "Should drop overlay in Failure status": + let treeCid = Cid.example - check: - mockRepoStore.delBlockCids == [testBe1.cid, testBe2.cid] + (await repo.putOverlay(treeCid, status = Failure.some, expiry = 200)).tryGet() + maintainer.start() - test "After deleting a block, subsequent timer callback should decrease offset by the number of deleted blocks": - mockClock.set(250) - blockMaintainer.start() await mockTimer.invokeCallback() + check (await repo.getOverlay(treeCid)).isErr - check mockRepoStore.delBlockCids == [testBe1.cid] + test "Should drop overlay left in Deleting status (crash leftover)": + let treeCid = Cid.example - # Because one block was deleted, the offset used in the next call should be 2 minus 1. + (await repo.putOverlay(treeCid, status = Deleting.some, expiry = 200)).tryGet() + + maintainer.start() await mockTimer.invokeCallback() - check: - mockRepoStore.getBeMaxNumber == 2 - mockRepoStore.getBeOffset == 1 + check (await repo.getOverlay(treeCid)).isErr - test "Should delete all blocks if expired, in two timer callbacks": - mockClock.set(500) - blockMaintainer.start() - await mockTimer.invokeCallback() - await mockTimer.invokeCallback() + test "Should skip overlay actively being deleted (runtime lock)": + let treeCid = Cid.example + + (await repo.putOverlay(treeCid, status = Deleting.some, expiry = 200)).tryGet() - check mockRepoStore.delBlockCids == [testBe1.cid, testBe2.cid, testBe3.cid] + # Simulate an active deletion by holding the lock + repo.deletingLock.incl(treeCid) - test "Iteration offset should loop": - blockMaintainer.start() + maintainer.start() await mockTimer.invokeCallback() - check mockRepoStore.getBeOffset == 0 + # Overlay should still exist because the lock prevented re-deletion + check (await repo.getOverlay(treeCid)).isOk + + repo.deletingLock.excl(treeCid) + + test "Should drop expired Pending overlay": + let treeCid = Cid.example + + (await repo.putOverlay(treeCid, status = Pending.some, expiry = 50)).tryGet() + + maintainer.start() await mockTimer.invokeCallback() - check mockRepoStore.getBeOffset == 2 + check (await repo.getOverlay(treeCid)).isErr + + test "Should not drop non-expired Pending overlay": + let treeCid = Cid.example + + (await repo.putOverlay(treeCid, status = Pending.some, expiry = 200)).tryGet() + + maintainer.start() await mockTimer.invokeCallback() - check mockRepoStore.getBeOffset == 0 - - test "Should handle new blocks": - proc invokeTimerManyTimes(): Future[void] {.async.} = - for i in countup(0, 10): - await mockTimer.invokeCallback() - - blockMaintainer.start() - await invokeTimerManyTimes() - - # no blocks have expired - check mockRepoStore.delBlockCids == [] - - mockClock.set(250) - await invokeTimerManyTimes() - # one block has expired - check mockRepoStore.delBlockCids == [testBe1.cid] - - # new blocks are added - let testBe4 = createTestExpiration(600) - let testBe5 = createTestExpiration(700) - mockRepoStore.testBlockExpirations.add(testBe4) - mockRepoStore.testBlockExpirations.add(testBe5) - - mockClock.set(500) - await invokeTimerManyTimes() - # All blocks have expired - check mockRepoStore.delBlockCids == [testBe1.cid, testBe2.cid, testBe3.cid] - - mockClock.set(650) - await invokeTimerManyTimes() - # First new block has expired - check mockRepoStore.delBlockCids == - [testBe1.cid, testBe2.cid, testBe3.cid, testBe4.cid] - - mockClock.set(750) - await invokeTimerManyTimes() - # Second new block has expired - check mockRepoStore.delBlockCids == - [testBe1.cid, testBe2.cid, testBe3.cid, testBe4.cid, testBe5.cid] + + check (await repo.getOverlay(treeCid)).isOk diff --git a/tests/archivist/stores/testrepostore.nim b/tests/archivist/stores/testrepostore.nim deleted file mode 100644 index a3ee2c0a..00000000 --- a/tests/archivist/stores/testrepostore.nim +++ /dev/null @@ -1,505 +0,0 @@ -import std/os -import std/strutils -import std/sequtils - -import pkg/questionable -import pkg/questionable/results - -import pkg/chronos -import pkg/stew/byteutils -import pkg/datastore - -import pkg/archivist/stores/cachestore -import pkg/archivist/chunker -import pkg/archivist/stores -import pkg/archivist/stores/repostore/operations -import pkg/archivist/blocktype as bt -import pkg/archivist/clock -import pkg/archivist/utils/safeasynciter -import pkg/archivist/merkletree/archivist - -import ../../asynctest -import ../helpers -import ../helpers/mockclock -import ../examples -import ./commonstoretests - -suite "Test RepoStore start/stop": - var - repoDs: Datastore - metaDs: Datastore - - setup: - repoDs = SQLiteDatastore.new(Memory).tryGet() - metaDs = SQLiteDatastore.new(Memory).tryGet() - - test "Should set started flag once started": - let repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = 200'nb) - await repo.start() - check repo.started - - test "Should set started flag to false once stopped": - let repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = 200'nb) - await repo.start() - await repo.stop() - check not repo.started - - test "Should allow start to be called multiple times": - let repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = 200'nb) - await repo.start() - await repo.start() - check repo.started - - test "Should allow stop to be called multiple times": - let repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = 200'nb) - await repo.stop() - await repo.stop() - check not repo.started - -asyncchecksuite "RepoStore": - var - repoDs: Datastore - metaDs: Datastore - mockClock: MockClock - - repo: RepoStore - - let now: SecondsSince1970 = 123 - - setup: - repoDs = SQLiteDatastore.new(Memory).tryGet() - metaDs = SQLiteDatastore.new(Memory).tryGet() - mockClock = MockClock.new() - mockClock.set(now) - - repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 200'nb) - - teardown: - (await repoDs.close()).tryGet - (await metaDs.close()).tryGet - - proc createTestBlock(size: int): bt.Block = - bt.Block.new('a'.repeat(size).toBytes).tryGet() - - test "Should update current used bytes on block put": - let blk = createTestBlock(200) - - check repo.quotaUsedBytes == 0'nb - (await repo.putBlock(blk)).tryGet - - check: - repo.quotaUsedBytes == 200'nb - - test "Should update current used bytes on block delete": - let blk = createTestBlock(100) - - check repo.quotaUsedBytes == 0'nb - (await repo.putBlock(blk)).tryGet - check repo.quotaUsedBytes == 100'nb - - (await repo.delBlock(blk.cid)).tryGet - - check: - repo.quotaUsedBytes == 0'nb - - test "Should not update current used bytes if block exist": - let blk = createTestBlock(100) - - check repo.quotaUsedBytes == 0'nb - (await repo.putBlock(blk)).tryGet - check repo.quotaUsedBytes == 100'nb - - # put again - (await repo.putBlock(blk)).tryGet - check repo.quotaUsedBytes == 100'nb - - test "Should fail storing passed the quota": - let blk = createTestBlock(300) - - check repo.totalUsed == 0'nb - expect QuotaNotEnoughError: - (await repo.putBlock(blk)).tryGet - - test "Should reserve bytes": - let blk = createTestBlock(100) - - check repo.totalUsed == 0'nb - (await repo.putBlock(blk)).tryGet - check repo.totalUsed == 100'nb - - (await repo.reserve(100'nb)).tryGet - - check: - repo.totalUsed == 200'nb - repo.quotaUsedBytes == 100'nb - repo.quotaReservedBytes == 100'nb - - test "Should not reserve bytes over max quota": - let blk = createTestBlock(100) - - check repo.totalUsed == 0'nb - (await repo.putBlock(blk)).tryGet - check repo.totalUsed == 100'nb - - expect QuotaNotEnoughError: - (await repo.reserve(101'nb)).tryGet - - check: - repo.totalUsed == 100'nb - repo.quotaUsedBytes == 100'nb - repo.quotaReservedBytes == 0'nb - - test "Should release bytes": - discard createTestBlock(100) - - check repo.totalUsed == 0'nb - (await repo.reserve(100'nb)).tryGet - check repo.totalUsed == 100'nb - - (await repo.release(100'nb)).tryGet - - check: - repo.totalUsed == 0'nb - repo.quotaUsedBytes == 0'nb - repo.quotaReservedBytes == 0'nb - - test "Should not release bytes less than quota": - check repo.totalUsed == 0'nb - (await repo.reserve(100'nb)).tryGet - check repo.totalUsed == 100'nb - - expect RangeDefect: - (await repo.release(101'nb)).tryGet - - check: - repo.totalUsed == 100'nb - repo.quotaUsedBytes == 0'nb - repo.quotaReservedBytes == 100'nb - - proc getExpirations(): Future[seq[BlockExpiration]] {.async.} = - let iter = (await repo.getBlockExpirations(100, 0)).tryGet() - - var res = newSeq[BlockExpiration]() - for fut in iter: - if be =? (await fut): - res.add(be) - res - - test "Should store block expiration timestamp": - let - duration = 10.seconds - blk = createTestBlock(100) - - let expectedExpiration = BlockExpiration(cid: blk.cid, expiry: now + 10) - - (await repo.putBlock(blk, duration.some)).tryGet - - let expirations = await getExpirations() - - check: - expectedExpiration in expirations - - test "Should store block with default expiration timestamp when not provided": - let blk = createTestBlock(100) - - let expectedExpiration = - BlockExpiration(cid: blk.cid, expiry: now + DefaultBlockTtl.seconds) - - (await repo.putBlock(blk)).tryGet - - let expirations = await getExpirations() - - check: - expectedExpiration in expirations - - test "Should refuse update expiry with negative timestamp": - let - blk = createTestBlock(100) - expectedExpiration = BlockExpiration(cid: blk.cid, expiry: now + 10) - - (await repo.putBlock(blk, some 10.seconds)).tryGet - - let expirations = await getExpirations() - - check: - expectedExpiration in expirations - - expect ValueError: - (await repo.ensureExpiry(blk.cid, -1)).tryGet - - expect ValueError: - (await repo.ensureExpiry(blk.cid, 0)).tryGet - - test "Should fail when updating expiry of non-existing block": - let blk = createTestBlock(100) - - expect BlockNotFoundError: - (await repo.ensureExpiry(blk.cid, 10)).tryGet - - test "Should update block expiration timestamp when new expiration is farther": - let - blk = createTestBlock(100) - expectedExpiration = BlockExpiration(cid: blk.cid, expiry: now + 10) - updatedExpectedExpiration = BlockExpiration(cid: blk.cid, expiry: now + 20) - - (await repo.putBlock(blk, some 10.seconds)).tryGet - - let expirations = await getExpirations() - - check: - expectedExpiration in expirations - - (await repo.ensureExpiry(blk.cid, now + 20)).tryGet - - let updatedExpirations = await getExpirations() - - check: - expectedExpiration notin updatedExpirations - updatedExpectedExpiration in updatedExpirations - - test "Should not update block expiration timestamp when current expiration is farther then new one": - let - blk = createTestBlock(100) - expectedExpiration = BlockExpiration(cid: blk.cid, expiry: now + 10) - updatedExpectedExpiration = BlockExpiration(cid: blk.cid, expiry: now + 5) - - (await repo.putBlock(blk, some 10.seconds)).tryGet - - let expirations = await getExpirations() - - check: - expectedExpiration in expirations - - (await repo.ensureExpiry(blk.cid, now + 5)).tryGet - - let updatedExpirations = await getExpirations() - - check: - expectedExpiration in updatedExpirations - updatedExpectedExpiration notin updatedExpirations - - test "delBlock should remove expiration metadata": - let - blk = createTestBlock(100) - expectedKey = Key.init("meta/ttl/" & $blk.cid).tryGet - - (await repo.putBlock(blk, 10.seconds.some)).tryGet - (await repo.delBlock(blk.cid)).tryGet - - let expirations = await getExpirations() - - check: - expirations.len == 0 - - test "Should retrieve block expiration information": - proc unpack( - beIter: Future[?!SafeAsyncIter[BlockExpiration]] - ): Future[seq[BlockExpiration]] {.async: (raises: [CatchableError]).} = - var expirations = newSeq[BlockExpiration](0) - without iter =? (await beIter), err: - return expirations - for beFut in toSeq(iter): - if value =? (await beFut): - expirations.add(value) - return expirations - - let - duration = 10.seconds - blk1 = createTestBlock(10) - blk2 = createTestBlock(11) - blk3 = createTestBlock(12) - - let expectedExpiration: SecondsSince1970 = now + 10 - - proc assertExpiration(be: BlockExpiration, expectedBlock: bt.Block) = - check: - be.cid == expectedBlock.cid - be.expiry == expectedExpiration - - (await repo.putBlock(blk1, duration.some)).tryGet - (await repo.putBlock(blk2, duration.some)).tryGet - (await repo.putBlock(blk3, duration.some)).tryGet - - let - blockExpirations1 = - await unpack(repo.getBlockExpirations(maxNumber = 2, offset = 0)) - blockExpirations2 = - await unpack(repo.getBlockExpirations(maxNumber = 2, offset = 2)) - - check blockExpirations1.len == 2 - assertExpiration(blockExpirations1[0], blk2) - assertExpiration(blockExpirations1[1], blk1) - - check blockExpirations2.len == 1 - assertExpiration(blockExpirations2[0], blk3) - - test "should put empty blocks": - let blk = Cid.example.emptyBlock.tryGet() - check (await repo.putBlock(blk)).isOk - - test "should get empty blocks": - let blk = Cid.example.emptyBlock.tryGet() - - let got = await repo.getBlock(blk.cid) - check got.isOk - check got.get.cid == blk.cid - - test "should delete empty blocks": - let blk = Cid.example.emptyBlock.tryGet() - check (await repo.delBlock(blk.cid)).isOk - - test "should have empty block": - let blk = Cid.example.emptyBlock.tryGet() - - let has = await repo.hasBlock(blk.cid) - check has.isOk - check has.get - - test "should set the reference count for orphan blocks to 0": - let blk = Block.example(size = 200) - (await repo.putBlock(blk)).tryGet() - check (await repo.blockRefCount(blk.cid)).tryGet() == 0.Natural - - test "should not allow non-orphan blocks to be deleted directly": - let - repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = - 1000'nb) - dataset = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) - blk = dataset[0] - (manifest, tree) = makeManifestAndTree(dataset).tryGet() - treeCid = tree.rootCid.tryGet() - proof = tree.getProof(0).tryGet() - - (await repo.putBlock(blk)).tryGet() - (await repo.putCidAndProof(treeCid, 0, blk.cid, proof)).tryGet() - - let err = (await repo.delBlock(blk.cid)).error() - check err.msg == - "Directly deleting a block that is part of a dataset is not allowed." - - test "should allow non-orphan blocks to be deleted by dataset reference": - let - repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = - 1000'nb) - dataset = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) - blk = dataset[0] - (manifest, tree) = makeManifestAndTree(dataset).tryGet() - treeCid = tree.rootCid.tryGet() - proof = tree.getProof(0).tryGet() - - (await repo.putBlock(blk)).tryGet() - (await repo.putCidAndProof(treeCid, 0, blk.cid, proof)).tryGet() - - (await repo.delBlock(treeCid, 0.Natural)).tryGet() - check not (await blk.cid in repo) - - test "should not delete a non-orphan block until it is deleted from all parent datasets": - let - repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = - 1000'nb) - blockPool = await makeRandomBlocks(datasetSize = 768, blockSize = 256'nb) - - let - dataset1 = @[blockPool[0], blockPool[1]] - dataset2 = @[blockPool[1], blockPool[2]] - - let sharedBlock = blockPool[1] - - let - (manifest1, tree1) = makeManifestAndTree(dataset1).tryGet() - treeCid1 = tree1.rootCid.tryGet() - (manifest2, tree2) = makeManifestAndTree(dataset2).tryGet() - treeCid2 = tree2.rootCid.tryGet() - - (await repo.putBlock(sharedBlock)).tryGet() - check (await repo.blockRefCount(sharedBlock.cid)).tryGet() == 0.Natural - - let - proof1 = tree1.getProof(1).tryGet() - proof2 = tree2.getProof(0).tryGet() - - (await repo.putCidAndProof(treeCid1, 1, sharedBlock.cid, proof1)).tryGet() - check (await repo.blockRefCount(sharedBlock.cid)).tryGet() == 1.Natural - - (await repo.putCidAndProof(treeCid2, 0, sharedBlock.cid, proof2)).tryGet() - check (await repo.blockRefCount(sharedBlock.cid)).tryGet() == 2.Natural - - (await repo.delBlock(treeCid1, 1.Natural)).tryGet() - check (await repo.blockRefCount(sharedBlock.cid)).tryGet() == 1.Natural - check (await sharedBlock.cid in repo) - - (await repo.delBlock(treeCid2, 0.Natural)).tryGet() - check not (await sharedBlock.cid in repo) - - test "should clear leaf metadata when block is deleted from dataset": - let - repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = - 1000'nb) - dataset = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) - blk = dataset[0] - (manifest, tree) = makeManifestAndTree(dataset).tryGet() - treeCid = tree.rootCid.tryGet() - proof = tree.getProof(1).tryGet() - - (await repo.putBlock(blk)).tryGet() - (await repo.putCidAndProof(treeCid, 0.Natural, blk.cid, proof)).tryGet() - - discard (await repo.getLeafMetadata(treeCid, 0.Natural)).tryGet() - - (await repo.delBlock(treeCid, 0.Natural)).tryGet() - - let err = (await repo.getLeafMetadata(treeCid, 0.Natural)).error() - check err of BlockNotFoundError - - test "should not fail when reinserting and deleting a previously deleted block (bug #1108)": - let - repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = - 1000'nb) - dataset = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) - blk = dataset[0] - (manifest, tree) = makeManifestAndTree(dataset).tryGet() - treeCid = tree.rootCid.tryGet() - proof = tree.getProof(1).tryGet() - - (await repo.putBlock(blk)).tryGet() - (await repo.putCidAndProof(treeCid, 0, blk.cid, proof)).tryGet() - - (await repo.delBlock(treeCid, 0.Natural)).tryGet() - (await repo.putBlock(blk)).tryGet() - (await repo.delBlock(treeCid, 0.Natural)).tryGet() - -commonBlockStoreTests( - "RepoStore Sql backend", - proc(): BlockStore = - BlockStore( - RepoStore.new( - SQLiteDatastore.new(Memory).tryGet(), - SQLiteDatastore.new(Memory).tryGet(), - clock = MockClock.new(), - ) - ), -) - -const path = currentSourcePath().parentDir / "test" - -proc before() {.async.} = - createDir(path) - -proc after() {.async.} = - removeDir(path) - -let depth = path.split(DirSep).len - -commonBlockStoreTests( - "RepoStore FS backend", - proc(): BlockStore = - BlockStore( - RepoStore.new( - FSDatastore.new(path, depth).tryGet(), - SQLiteDatastore.new(Memory).tryGet(), - clock = MockClock.new(), - ) - ), - before = before, - after = after, -) diff --git a/tests/archivist/testchunking.nim b/tests/archivist/testchunking.nim index bc6e8fc6..d469c4e8 100644 --- a/tests/archivist/testchunking.nim +++ b/tests/archivist/testchunking.nim @@ -1,4 +1,5 @@ import pkg/stew/byteutils +import pkg/questionable/results import pkg/archivist/chunker import pkg/archivist/logutils import pkg/chronos @@ -27,24 +28,24 @@ asyncchecksuite "Chunking": let contents = [1.byte, 2, 3, 4, 5, 6, 7, 8, 9, 0] proc reader( data: ChunkBuffer, len: int - ): Future[int] {.gcsafe, async: (raises: [ChunkerError, CancelledError]).} = + ): Future[?!int] {.gcsafe, async: (raises: [CancelledError]).} = let read = min(contents.len - offset, len) if read == 0: - return 0 + return success 0 copyMem(data, unsafeAddr contents[offset], read) offset += read - return read + return success read let chunker = Chunker.new(reader = reader, chunkSize = 2'nb) check: - (await chunker.getBytes()) == [1.byte, 2] - (await chunker.getBytes()) == [3.byte, 4] - (await chunker.getBytes()) == [5.byte, 6] - (await chunker.getBytes()) == [7.byte, 8] - (await chunker.getBytes()) == [9.byte, 0] - (await chunker.getBytes()) == [] + (await chunker.getBytes()).tryGet() == [1.byte, 2] + (await chunker.getBytes()).tryGet() == [3.byte, 4] + (await chunker.getBytes()).tryGet() == [5.byte, 6] + (await chunker.getBytes()).tryGet() == [7.byte, 8] + (await chunker.getBytes()).tryGet() == [9.byte, 0] + (await chunker.getBytes()).tryGet() == [] chunker.offset == offset test "should chunk LPStream": @@ -59,12 +60,12 @@ asyncchecksuite "Chunking": let writerFut = writer() check: - (await chunker.getBytes()) == [1.byte, 2] - (await chunker.getBytes()) == [3.byte, 4] - (await chunker.getBytes()) == [5.byte, 6] - (await chunker.getBytes()) == [7.byte, 8] - (await chunker.getBytes()) == [9.byte, 0] - (await chunker.getBytes()) == [] + (await chunker.getBytes()).tryGet() == [1.byte, 2] + (await chunker.getBytes()).tryGet() == [3.byte, 4] + (await chunker.getBytes()).tryGet() == [5.byte, 6] + (await chunker.getBytes()).tryGet() == [7.byte, 8] + (await chunker.getBytes()).tryGet() == [9.byte, 0] + (await chunker.getBytes()).tryGet() == [] chunker.offset == 10 await writerFut @@ -77,7 +78,7 @@ asyncchecksuite "Chunking": var data: seq[byte] while true: - let buff = await fileChunker.getBytes() + let buff = (await fileChunker.getBytes()).tryGet() if buff.len <= 0: break @@ -97,16 +98,24 @@ asyncchecksuite "Chunking": discard (await chunker.getBytes()) test "stream should forward LPStreamError": - try: - await raiseStreamException(newException(LPStreamError, "test error")) - except ChunkerError as exc: - check exc.parent of LPStreamError - except CatchableError as exc: - checkpoint("Unexpected error: " & exc.msg) - fail() + let stream = CrashingStreamWrapper.new() + let chunker = LPStreamChunker.new(stream = stream, chunkSize = 2'nb) + + stream.toRaise = proc(): void {.raises: [CancelledError, LPStreamError].} = + raise newException(LPStreamError, "test error") + + let res = await chunker.getBytes() + check res.isErr + check res.error of LPStreamError test "stream should catch LPStreamEOFError": - await raiseStreamException(newException(LPStreamEOFError, "test error")) + let stream = CrashingStreamWrapper.new() + let chunker = LPStreamChunker.new(stream = stream, chunkSize = 2'nb) + + stream.toRaise = proc(): void {.raises: [CancelledError, LPStreamError].} = + raise newException(LPStreamEOFError, "test error") + + check (await chunker.getBytes()).tryGet() == [] test "stream should forward CancelledError": expect CancelledError: diff --git a/tests/archivist/testerasure.nim b/tests/archivist/testerasure.nim index eaebab90..80aa0c6c 100644 --- a/tests/archivist/testerasure.nim +++ b/tests/archivist/testerasure.nim @@ -12,39 +12,37 @@ import pkg/archivist/rng import pkg/archivist/utils import pkg/archivist/indexingstrategy import pkg/taskpools +import pkg/kvstore import ../asynctest import ./helpers import ./examples suite "Erasure encode/decode": - const BlockSize = 1024'nb + const BlockSize = 128'nb const dataSetSize = BlockSize * 123 # weird geometry var rng: Rng var chunker: Chunker var manifest: Manifest - var store: BlockStore + var store: RepoStore var erasure: Erasure - let repoTmp = TempLevelDb.new() - let metaTmp = TempLevelDb.new() - var taskpool: Taskpool + var tp: Taskpool setup: + tp = Taskpool.new(num_threads = 4) let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() + repoDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() + metaDs = SQLiteKVStore.new(SqliteMemory, tp).tryGet() rng = Rng.instance() chunker = RandomChunker.new(rng, size = dataSetSize, chunkSize = BlockSize) store = RepoStore.new(repoDs, metaDs) - taskpool = Taskpool.new() - erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, taskpool) - manifest = await storeDataGetManifest(store, chunker) + erasure = Erasure.new(store, store, leoEncoderProvider, leoDecoderProvider, tp) + manifest = (await storeDataGetManifest(store, chunker)).tryGet() teardown: - await repoTmp.destroyDb() - await metaTmp.destroyDb() - taskpool.shutdown() + await store.close() + tp.shutdown() proc encode(buffers, parity: int): Future[Manifest] {.async.} = let encoded = @@ -201,9 +199,7 @@ suite "Erasure encode/decode": let encoded = await encode(buffers, parity) - blocks = collect: - for i in 0 .. encoded.blocksCount: - i + blocks = toSeq(0 .. encoded.blocksCount) # loose M parity (all!) symbols/blocks from the dataset for b in blocks[^(encoded.steps * encoded.ecM) ..^ 1]: @@ -241,7 +237,7 @@ suite "Erasure encode/decode": # create random data and store it blockSize = rng.sample(@[1, 2, 4, 8, 16, 32, 64].mapIt(it.KiBs)) chunker = RandomChunker.new(rng, size = datasetSize, chunkSize = blockSize) - manifest = await storeDataGetManifest(store, chunker) + manifest = (await storeDataGetManifest(store, chunker)).tryGet() manifests.add(manifest) # encode the data concurrently encodeTasks.add(erasure.encode(manifest, ecK, ecM)) @@ -292,7 +288,7 @@ suite "Erasure encode/decode": let chunker = RandomChunker.new(rng, size = datasetSize, chunkSize = blockSize) - manifest = await storeDataGetManifest(store, chunker) + manifest = (await storeDataGetManifest(store, chunker)).tryGet() encoded = (await erasure.encode(manifest, ecK, ecM)).tryGet() decoded = (await erasure.decode(encoded)).tryGet() @@ -313,7 +309,7 @@ suite "Erasure encode/decode": let recovered = new seq[seq[byte]] let cancelledTaskParity = new seq[seq[byte]] let cancelledTaskRecovered = new seq[seq[byte]] - data[] = newSeqWith(blocksLen, await chunker.getBytes()) + data[] = newSeqWith(blocksLen, (await chunker.getBytes()).tryGet()) parity[] = newSeqWith(10, newSeqWith(BlockSize.int, 0'u8)) cancelledTaskParity[] = newSeqWith(10, newSeqWith(BlockSize.int, 0'u8)) recovered[] = newSeqWith(blocksLen, newSeqWith(BlockSize.int, 0'u8)) diff --git a/tests/archivist/testmarketplacestorage.nim b/tests/archivist/testmarketplacestorage.nim index c97e5c93..6ca7961c 100644 --- a/tests/archivist/testmarketplacestorage.nim +++ b/tests/archivist/testmarketplacestorage.nim @@ -2,7 +2,7 @@ import std/os import std/times import pkg/chronos import pkg/taskpools -import pkg/datastore/typedds +import pkg/kvstore import pkg/archivist/marketplace import pkg/archivist/marketplacestorage import pkg/archivist/marketplace/timestamps @@ -11,7 +11,6 @@ import pkg/archivist/chunker import pkg/archivist/erasure import pkg/archivist/slots import pkg/archivist/stores -import pkg/archivist/indexingstrategy import ../asynctest import ./node/tempnode import ./helpers @@ -29,7 +28,6 @@ suite "Marketplace storage interface implementation": await temporary.destroy() proc storeVerifiableData(): Future[Cid] {.async.} = - let node = temporary.node let localStore = temporary.localStore let networkStore = temporary.networkStore let path = currentSourcePath().parentDir @@ -39,46 +37,45 @@ suite "Marketplace storage interface implementation": let encoder = leoEncoderProvider let decoder = leoDecoderProvider let taskpool = TaskPool.new() - let erasure = Erasure.new(networkStore, encoder, decoder, taskpool) - let manifest = await storeDataGetManifest(localStore, chunker) + let erasure = Erasure.new(networkStore, localStore, encoder, decoder, taskpool) + let manifest = !await storeDataGetManifest(localStore, chunker) let protected = !await erasure.encode(manifest, 3, 2) - let builder = !Poseidon2Builder.new(localStore, protected) + let builder = !Poseidon2Builder.new(networkStore, localStore, protected) verifiable = !await builder.buildManifest() - let cid = (!await node.storeManifest(verifiable)).cid + let cid = (!await localStore.storeManifest(verifiable)).cid file.close() cid - proc checkBlockExpiry(cid: Cid, expiry: int64) {.async.} = - let localStore = temporary.localStore - let key = !createBlockExpirationMetadataKey(cid) - let metadata = !await get[BlockMetaData](localStore.metaDs, key) + proc checkOverlayExpiry(cid: Cid, expiry: int64) {.async.} = + let + localStore = temporary.localStore + manifestBlock = !await localStore.getBlock(cid) + manifest = !Manifest.decode(manifestBlock) + metadata = !await localStore.getOverlay(manifest.treeCid) + check metadata.expiry == expiry proc checkSlotExpiry(cid: Cid, slotIndex: uint64, expiry: int64) {.async.} = - let node = temporary.node - let localStore = temporary.localStore - let manifest = !await node.fetchManifest(cid) - let strategy = manifest.verifiableStrategy - let indexer = strategy.init(0, manifest.blocksCount - 1, manifest.numSlots) - for index in indexer.getIndices(slotIndex.int): - let blockCid = !await localStore.getCid(manifest.treeCid, index) - await checkBlockExpiry(blockCid, expiry) + # TODO: updateSlotExpiry currently updates the entire dataset expiry, + # not per-slot expiry, so we just check the manifest overlay + discard slotIndex + await checkOverlayExpiry(cid, expiry) test "updates expiry of slot blocks": let cid = await storeVerifiableData() - let expiry = getTime().toUnix + DefaultBlockTtl.seconds + 42 + let expiry = getTime().toUnix + 42 !await storage.updateSlotExpiry(cid, 0, StorageTimestamp.init(expiry)) await checkSlotExpiry(cid, 0, expiry) test "updates expiry of dataset manifest": let cid = await storeVerifiableData() - let expiry = getTime().toUnix + DefaultBlockTtl.seconds + 42 + let expiry = getTime().toUnix + 42 !await storage.updateSlotExpiry(cid, 0, StorageTimestamp.init(expiry)) - await checkBlockExpiry(cid, expiry) + await checkOverlayExpiry(cid, expiry) test "rejects manifest with incorrect slotSize": let cid = await storeVerifiableData() - let expiry = getTime().toUnix + DefaultBlockTtl.seconds + 42 + let expiry = getTime().toUnix + 42 let response = await storage.storeSlot( cid, 0, verifiable.slotSize.uint64 - 1, StorageTimestamp.init(expiry), false ) @@ -88,7 +85,7 @@ suite "Marketplace storage interface implementation": test "storing a slot updates the expiry of the slot blocks": let cid = await storeVerifiableData() - let expiry = getTime().toUnix + DefaultBlockTtl.seconds + 42 + let expiry = getTime().toUnix + 42 !await storage.storeSlot( cid, 0, verifiable.slotSize.uint64, StorageTimestamp.init(expiry), false ) @@ -96,8 +93,46 @@ suite "Marketplace storage interface implementation": test "storing a slot updates the expiry of the dataset manifest": let cid = await storeVerifiableData() - let expiry = getTime().toUnix + DefaultBlockTtl.seconds + 42 + let expiry = getTime().toUnix + 42 !await storage.storeSlot( cid, 0, verifiable.slotSize.uint64, StorageTimestamp.init(expiry), false ) - await checkBlockExpiry(cid, expiry) + await checkOverlayExpiry(cid, expiry) + + test "deleteSlot marks overlay as Failure": + let + cid = await storeVerifiableData() + localStore = temporary.localStore + manifestBlock = !await localStore.getBlock(cid) + manifest = !Manifest.decode(manifestBlock) + expiry = getTime().toUnix + 42 + + !await storage.storeSlot( + cid, 0, verifiable.slotSize.uint64, StorageTimestamp.init(expiry), false + ) + + !await storage.deleteSlot(cid, 0) + + # Check that slot overlay status is now Failure + let slotCid = manifest.slotRoots[0] + let metadata = !await localStore.getOverlay(slotCid) + check metadata.status == OverlayStatus.Failure + + test "deleteSlot marks overlay as Failure for different slot indices": + let + cid = await storeVerifiableData() + localStore = temporary.localStore + manifestBlock = !await localStore.getBlock(cid) + manifest = !Manifest.decode(manifestBlock) + expiry = getTime().toUnix + 42 + + !await storage.storeSlot( + cid, 0, verifiable.slotSize.uint64, StorageTimestamp.init(expiry), false + ) + + !await storage.deleteSlot(cid, 1) + + # Check that slot overlay status is now Failure + let slotCid = manifest.slotRoots[1] + let metadata = !await localStore.getOverlay(slotCid) + check metadata.status == OverlayStatus.Failure diff --git a/tests/archivist/teststorestream.nim b/tests/archivist/teststorestream.nim index 7b238d62..1e4864e5 100644 --- a/tests/archivist/teststorestream.nim +++ b/tests/archivist/teststorestream.nim @@ -1,4 +1,6 @@ import pkg/chronos +import pkg/kvstore +import pkg/taskpools import pkg/archivist/[streams, stores, indexingstrategy, manifest, blocktype as bt] @@ -9,8 +11,9 @@ import ./helpers asyncchecksuite "StoreStream": var manifest: Manifest - store: BlockStore + store: RepoStore stream: StoreStream + tp: Taskpool # Check that `buf` contains `size` bytes with values start, start+1... proc sequentialBytes(buf: seq[byte], size: int, start: int): bool = @@ -31,12 +34,19 @@ asyncchecksuite "StoreStream": teardown: await stream.close() + tp.shutdown() setup: - store = CacheStore.new() - manifest = await storeDataGetManifest( - store, MockChunker.new(dataset = data, chunkSize = chunkSize) + tp = Taskpool.new(num_threads = 4) + store = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), ) + manifest = ( + await storeDataGetManifest( + store, MockChunker.new(dataset = data, chunkSize = chunkSize) + ) + ).tryGet() stream = StoreStream.new(store, manifest) test "Read all blocks < blockSize": @@ -95,35 +105,51 @@ asyncchecksuite "StoreStream": check sequentialBytes(buf, 15, 0) suite "StoreStream - Size Tests": - var stream: StoreStream + var + stream: StoreStream + tp: Taskpool + + setup: + tp = Taskpool.new(num_threads = 4) teardown: await stream.close() + tp.shutdown() test "Should return dataset size as stream size": - let manifest = Manifest.new( - treeCid = Cid.example, datasetSize = 80.NBytes, blockSize = 10.NBytes - ) + let + manifest = Manifest.new( + treeCid = Cid.example, datasetSize = 80.NBytes, blockSize = 10.NBytes + ) + store = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) - stream = StoreStream.new(CacheStore.new(), manifest) + stream = StoreStream.new(store, manifest) check stream.size == 80 test "Should not count parity/padding bytes as part of stream size": - let protectedManifest = Manifest.new( - treeCid = Cid.example, - datasetSize = 120.NBytes, # size including parity bytes - blockSize = 10.NBytes, - version = CIDv1, - hcodec = Sha256HashCodec, - codec = BlockCodec, - ecK = 2, - ecM = 1, - originalTreeCid = Cid.example, - originalDatasetSize = 80.NBytes, # size without parity bytes - strategy = StrategyType.SteppedStrategy, - ) - - stream = StoreStream.new(CacheStore.new(), protectedManifest) + let + protectedManifest = Manifest.new( + treeCid = Cid.example, + datasetSize = 120.NBytes, # size including parity bytes + blockSize = 10.NBytes, + version = CIDv1, + hcodec = Sha256HashCodec, + codec = BlockCodec, + ecK = 2, + ecM = 1, + originalTreeCid = Cid.example, + originalDatasetSize = 80.NBytes, # size without parity bytes + strategy = StrategyType.SteppedStrategy, + ) + store = RepoStore.new( + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + SQLiteKVStore.new(SqliteMemory, tp).tryGet(), + ) + + stream = StoreStream.new(store, protectedManifest) check stream.size == 80 diff --git a/tests/archivist/utils/testutils.nim b/tests/archivist/utils/testutils.nim index 5b689472..1bad8b8f 100644 --- a/tests/archivist/utils/testutils.nim +++ b/tests/archivist/utils/testutils.nim @@ -1,4 +1,5 @@ import pkg/unittest2 +import pkg/stew/bitseqs import pkg/archivist/utils @@ -13,3 +14,90 @@ suite "parseDuration": check res == minutes(7) # 1 shl 30, forced binary metric check parseDuration("3d", res) == 2 # '/' stops parse check res == days(3) # 1 shl 30, forced binary metric + +suite "combineSafe": + test "should combine same-length BitSeqs": + var a = BitSeq.init(4) + a[0] = true + a[2] = true + var b = BitSeq.init(4) + b[1] = true + b[3] = true + + a.combineSafe(b) + check a.len == 4 + check a[0] == true + check a[1] == true + check a[2] == true + check a[3] == true + + test "should combine when tgt is longer": + var tgt = BitSeq.init(8) + tgt[0] = true + tgt[5] = true + var src = BitSeq.init(3) + src[1] = true + src[2] = true + + tgt.combineSafe(src) + check tgt.len == 8 + check tgt[0] == true + check tgt[1] == true + check tgt[2] == true + check tgt[3] == false + check tgt[4] == false + check tgt[5] == true + check tgt[6] == false + check tgt[7] == false + + test "should combine when tgt is shorter": + var tgt = BitSeq.init(3) + tgt[0] = true + var src = BitSeq.init(8) + src[1] = true + src[5] = true + src[7] = true + + tgt.combineSafe(src) + check tgt.len == 8 + check tgt[0] == true + check tgt[1] == true + check tgt[2] == false + check tgt[3] == false + check tgt[4] == false + check tgt[5] == true + check tgt[6] == false + check tgt[7] == true + + test "should handle empty src": + var tgt = BitSeq.init(4) + tgt[0] = true + let src = BitSeq.init(0) + + tgt.combineSafe(src) + check tgt.len == 4 + check tgt[0] == true + + test "should handle empty tgt": + var tgt = BitSeq.init(0) + var src = BitSeq.init(3) + src[1] = true + + tgt.combineSafe(src) + check tgt.len == 3 + check tgt[0] == false + check tgt[1] == true + check tgt[2] == false + + test "should not set marker bit as data": + # Verify that src's SSZ marker bit does not leak as a data bit in tgt + var tgt = BitSeq.init(8) + var src = BitSeq.init(3) + src[0] = true + + tgt.combineSafe(src) + check tgt.len == 8 + check tgt[0] == true + check tgt[1] == false + check tgt[2] == false + check tgt[3] == false # src's marker was at bit 3, must not appear as data diff --git a/tests/examples.nim b/tests/examples.nim index 99c36649..ebffbbeb 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -106,7 +106,7 @@ proc example*(_: type RandomChunker, blocks: int): Future[seq[byte]] {.async.} = rng, size = DefaultBlockSize * blocks.NBytes, chunkSize = DefaultBlockSize ) var data: seq[byte] - while (let moar = await chunker.getBytes(); moar != []): + while (let moar = (await chunker.getBytes()).tryGet(); moar != []): data.add moar return data diff --git a/tests/helpers.nim b/tests/helpers.nim index b48b787e..2e456a14 100644 --- a/tests/helpers.nim +++ b/tests/helpers.nim @@ -1,10 +1,9 @@ import helpers/multisetup import helpers/trackers -import helpers/templeveldb import std/times import std/sequtils, chronos -export multisetup, trackers, templeveldb +export multisetup, trackers ### taken from libp2p errorhelpers.nim proc allFuturesThrowing*(args: varargs[FutureBase]): Future[void] = diff --git a/tests/helpers/templeveldb.nim b/tests/helpers/templeveldb.nim deleted file mode 100644 index dbc53bb4..00000000 --- a/tests/helpers/templeveldb.nim +++ /dev/null @@ -1,29 +0,0 @@ -import os -import std/monotimes -import pkg/datastore -import pkg/chronos -import pkg/questionable/results - -type TempLevelDb* = ref object - currentPath: string - ds: LevelDbDatastore - -var number = 0 - -proc newDb*(self: TempLevelDb): Datastore = - if self.currentPath.len > 0: - raiseAssert("TempLevelDb already active.") - self.currentPath = getTempDir() / "templeveldb" / $number / $getMonoTime() - inc number - createDir(self.currentPath) - self.ds = LevelDbDatastore.new(self.currentPath).tryGet() - return self.ds - -proc destroyDb*(self: TempLevelDb): Future[void] {.async.} = - if self.currentPath.len == 0: - raiseAssert("TempLevelDb not active.") - try: - (await self.ds.close()).tryGet() - finally: - removeDir(self.currentPath) - self.currentPath = "" diff --git a/tests/integration/1_minute/testblockmaintenance.nim b/tests/integration/1_minute/testblockmaintenance.nim index f11072ee..e3b8fb8c 100644 --- a/tests/integration/1_minute/testblockmaintenance.nim +++ b/tests/integration/1_minute/testblockmaintenance.nim @@ -9,7 +9,7 @@ suite "Block maintenance": setup: testbed = await Testbed.start() - node = await testbed.node.blockTtl(5).blockMaintenanceInterval(1).start() + node = await testbed.node.overlayTtl(5).overlayMaintenanceInterval(1).start() teardown: await testbed.stop() diff --git a/tests/integration/5_minutes/testdatasets.nim b/tests/integration/5_minutes/testdatasets.nim index 80d9c3cb..f388d7ff 100644 --- a/tests/integration/5_minutes/testdatasets.nim +++ b/tests/integration/5_minutes/testdatasets.nim @@ -2,7 +2,10 @@ import std/json import std/sequtils import std/strutils import pkg/asynctest/chronos/unittest2 +import pkg/libp2p/cid +import pkg/libp2p/multihash import pkg/questionable +import pkg/archivist/archivisttypes import ../../testbed suite "Node datasets": @@ -48,6 +51,12 @@ suite "Node datasets": except HttpError as error: check "404" in error.msg - test "node allows deletion of absent dataset": - let cid = "zb2rhe5P4gXftAwvA4eXQ5HJwsER2owDyS9sKaQRRVQPn93bA" - await testbed.api(node).delete(cid) + test "node returns 404 when deleting absent dataset": + let + mhash = MultiHash.digest("sha2-256", @[0'u8]).tryGet() + cid = Cid.init(CIDv1, ManifestCodec, mhash).tryGet() + try: + await testbed.api(node).delete($cid) + fail() + except HttpError as error: + check "404" in error.msg diff --git a/tests/testbed.nim b/tests/testbed.nim index 45ae7593..3f191177 100644 --- a/tests/testbed.nim +++ b/tests/testbed.nim @@ -42,8 +42,8 @@ export node.provider export node.availability export node.failProofs export node.storageQuota -export node.blockTtl -export node.blockMaintenanceInterval +export node.overlayTtl +export node.overlayMaintenanceInterval export node.waitForOutput export node.start diff --git a/tests/testbed/builders/node.nim b/tests/testbed/builders/node.nim index 6a743bc9..05d613cf 100644 --- a/tests/testbed/builders/node.nim +++ b/tests/testbed/builders/node.nim @@ -39,8 +39,8 @@ type NodeBuilder = ref object circomGraph: ? ?string failProofs: ?int storageQuota: ?int - blockTtl: ?int - blockMaintenanceInterval: ?int + overlayTtl: ?int + overlayMaintenanceInterval: ?int waitForOutput: ?string setInitialAvailability: bool @@ -171,12 +171,12 @@ func storageQuota*(builder: NodeBuilder, quota: int): NodeBuilder = builder.storageQuota = some quota builder -func blockTtl*(builder: NodeBuilder, ttl: int): NodeBuilder = - builder.blockTtl = some ttl +func overlayTtl*(builder: NodeBuilder, ttl: int): NodeBuilder = + builder.overlayTtl = some ttl builder -func blockMaintenanceInterval*(builder: NodeBuilder, interval: int): NodeBuilder = - builder.blockMaintenanceInterval = some interval +func overlayMaintenanceInterval*(builder: NodeBuilder, interval: int): NodeBuilder = + builder.overlayMaintenanceInterval = some interval builder proc dataDirResolved(builder: NodeBuilder): string = @@ -273,10 +273,10 @@ proc start*(builder: NodeBuilder): Future[Node] {.async.} = arguments.add("--circom-graph=" & circomGraph) if quota =? builder.storageQuota: arguments.add("--storage-quota=" & $quota) - if blockTtl =? builder.blockTtl: - arguments.add("--block-ttl=" & $blockTtl) - if blockMaintenanceInterval =? builder.blockMaintenanceInterval: - arguments.add("--block-mi=" & $blockMaintenanceInterval) + if overlayTtl =? builder.overlayTtl: + arguments.add("--overlay-ttl=" & $overlayTtl) + if overlayMaintenanceInterval =? builder.overlayMaintenanceInterval: + arguments.add("--block-mi=" & $overlayMaintenanceInterval) let dataDir = builder.dataDirResolved let address = builder.apiBindAddressResolved let port = await builder.apiPortResolved diff --git a/vendor/nimble/archivistdht b/vendor/nimble/archivistdht index d44be375..71d4d3dd 160000 --- a/vendor/nimble/archivistdht +++ b/vendor/nimble/archivistdht @@ -1 +1 @@ -Subproject commit d44be375ed8417d7bf0b25499f3ad209c9db5c56 +Subproject commit 71d4d3dd4323a355936f5b78592578ffb9cb2cfb diff --git a/vendor/nimble/datastore b/vendor/nimble/datastore deleted file mode 160000 index dadeca24..00000000 --- a/vendor/nimble/datastore +++ /dev/null @@ -1 +0,0 @@ -Subproject commit dadeca2431ff2d48fe784642def73aa9656b3baa diff --git a/vendor/nimble/leveldbstatic b/vendor/nimble/leveldbstatic deleted file mode 160000 index 67f28676..00000000 --- a/vendor/nimble/leveldbstatic +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 67f2867611f686e3b3d45c3a17d7242b80b54559 diff --git a/vendor/nimble/metrics b/vendor/nimble/metrics index 11d0cddf..8107b559 160000 --- a/vendor/nimble/metrics +++ b/vendor/nimble/metrics @@ -1 +1 @@ -Subproject commit 11d0cddfb0e711aa2a8c75d1892ae24a64c299fc +Subproject commit 8107b55942851f5a542ee8209490945f3c941b75 diff --git a/vendor/nimble/nim-kvstore b/vendor/nimble/nim-kvstore new file mode 160000 index 00000000..f6075d63 --- /dev/null +++ b/vendor/nimble/nim-kvstore @@ -0,0 +1 @@ +Subproject commit f6075d63fcc0d8f5ac5cf76d9e593d3f63d5ecfa diff --git a/vendor/nimble/poseidon2 b/vendor/nimble/poseidon2 index 8f4e8cb3..43aee989 160000 --- a/vendor/nimble/poseidon2 +++ b/vendor/nimble/poseidon2 @@ -1 +1 @@ -Subproject commit 8f4e8cb3567c34dd4d1711fbb6032562d8f4c11d +Subproject commit 43aee9895c7cac411a0156253e1c2d6304e3ac09 diff --git a/vendor/nimble/threading b/vendor/nimble/threading new file mode 160000 index 00000000..ee77a27e --- /dev/null +++ b/vendor/nimble/threading @@ -0,0 +1 @@ +Subproject commit ee77a27eca7042a2b68061d0bcee691e5b56aa10