diff --git a/.gitmodules b/.gitmodules index 41f5ff81..21587106 100644 --- a/.gitmodules +++ b/.gitmodules @@ -83,7 +83,7 @@ url = https://github.com/durability-labs/nim-poseidon2 [submodule "vendor/nimble/libp2p"] path = vendor/nimble/libp2p - url = https://github.com/durability-labs/nim-libp2p + url = https://github.com/vacp2p/nim-libp2p [submodule "vendor/nimble/stint"] path = vendor/nimble/stint url = https://github.com/durability-labs/nim-stint @@ -131,7 +131,7 @@ url = https://github.com/status-im/nim-eth [submodule "vendor/nimble/json_rpc"] path = vendor/nimble/json_rpc - url = https://github.com/status-im/nim-json-rpc + url = https://github.com/durability-labs/nim-json-rpc [submodule "vendor/nimble/serde"] path = vendor/nimble/serde url = https://github.com/durability-labs/nim-serde @@ -159,3 +159,6 @@ [submodule "vendor/nimble/zippy"] path = vendor/nimble/zippy url = https://github.com/guzba/zippy +[submodule "vendor/nimble/lsquic"] + path = vendor/nimble/lsquic + url = https://github.com/vacp2p/nim-lsquic.git diff --git a/archivist/blockexchange/network/network.nim b/archivist/blockexchange/network/network.nim index 62516029..34c4366a 100644 --- a/archivist/blockexchange/network/network.nim +++ b/archivist/blockexchange/network/network.nim @@ -125,7 +125,10 @@ proc send*( except CatchableError as err: error "Error sending message", peer = id, msg = err.msg finally: - b.inflightSema.release() + try: + b.inflightSema.release() + except AsyncSemaphoreError as error: + raise newException(AssertionDefect, error.msg, error) proc handleWantList( b: BlockExcNetwork, peer: NetworkPeer, list: WantList @@ -356,10 +359,17 @@ proc new*( ## Create a new BlockExcNetwork instance ## + var inflightSema: AsyncSemaphore + if maxInflight == 0: + inflightSema = newAsyncSemaphore(1) + discard inflightSema.tryAcquire() + else: + inflightSema = newAsyncSemaphore(maxInFlight) + let self = BlockExcNetwork( switch: switch, getConn: connProvider, - inflightSema: newAsyncSemaphore(maxInflight), + inflightSema: inflightSema, maxInflight: maxInflight, ) diff --git a/archivist/libp2p/contentids_exts.nim b/archivist/libp2p/contentids_exts.nim new file mode 100644 index 00000000..9ef6fbb5 --- /dev/null +++ b/archivist/libp2p/contentids_exts.nim @@ -0,0 +1,8 @@ +const ContentIdsExts = [ + multiCodec("codex-root"), + multiCodec("codex-manifest"), + multiCodec("codex-block"), + multiCodec("codex-slot-root"), + multiCodec("codex-proving-root"), + multiCodec("codex-slot-cell"), +] diff --git a/archivist/libp2p/multicodec_exts.nim b/archivist/libp2p/multicodec_exts.nim new file mode 100644 index 00000000..b14fbbe6 --- /dev/null +++ b/archivist/libp2p/multicodec_exts.nim @@ -0,0 +1,11 @@ +const CodecExts = [ + ("poseidon2-alt_bn_128-sponge-r2", 0xCD10), # bn128 rate 2 sponge + ("poseidon2-alt_bn_128-merkle-2kb", 0xCD11), # bn128 2kb compress & merkleize + ("poseidon2-alt_bn_128-keyed-compress", 0xCD12), # bn128 keyed compress] + ("codex-manifest", 0xCD01), + ("codex-block", 0xCD02), + ("codex-root", 0xCD03), + ("codex-slot-root", 0xCD04), + ("codex-proving-root", 0xCD05), + ("codex-slot-cell", 0xCD06), +] diff --git a/archivist/libp2p/multihash_exts.nim b/archivist/libp2p/multihash_exts.nim new file mode 100644 index 00000000..e9477177 --- /dev/null +++ b/archivist/libp2p/multihash_exts.nim @@ -0,0 +1,40 @@ +import blscurve/bls_public_exports +import pkg/constantine/hashes +import poseidon2 + +proc sha2_256hash_constantine(data: openArray[byte], output: var openArray[byte]) = + # Using Constantine's SHA256 instead of mhash for optimal performance on 32-byte merkle node hashing + # See: https://github.com/codex-storage/nim-codex/issues/1162 + if len(output) > 0: + let digest = hashes.sha256.hash(data) + copyMem(addr output[0], addr digest[0], 32) + +proc poseidon2_sponge_rate2(data: openArray[byte], output: var openArray[byte]) = + if len(output) > 0: + var digest = poseidon2.Sponge.digest(data).toBytes() + copyMem(addr output[0], addr digest[0], uint(len(output))) + +proc poseidon2_merkle_2kb_sponge(data: openArray[byte], output: var openArray[byte]) = + if len(output) > 0: + var digest = poseidon2.SpongeMerkle.digest(data, 2048).toBytes() + copyMem(addr output[0], addr digest[0], uint(len(output))) + +const Sha2256MultiHash* = MHash( + mcodec: multiCodec("sha2-256"), + size: sha256.sizeDigest, + coder: sha2_256hash_constantine, +) +const HashExts = [ + # override sha2-256 hash function + Sha2256MultiHash, + MHash( + mcodec: multiCodec("poseidon2-alt_bn_128-sponge-r2"), + size: 32, + coder: poseidon2_sponge_rate2, + ), + MHash( + mcodec: multiCodec("poseidon2-alt_bn_128-merkle-2kb"), + size: 32, + coder: poseidon2_merkle_2kb_sponge, + ), +] diff --git a/archivist/marketplace/contracts/marketplacecontract.nim b/archivist/marketplace/contracts/marketplacecontract.nim index a43aa0a6..fb2873d1 100644 --- a/archivist/marketplace/contracts/marketplacecontract.nim +++ b/archivist/marketplace/contracts/marketplacecontract.nim @@ -15,6 +15,7 @@ export requests type MarketplaceContract* = ref object of Contract + RequestId = requests.RequestId Marketplace_RepairRewardPercentageTooHigh* = object of SolidityError Marketplace_SlashPercentageTooHigh* = object of SolidityError diff --git a/archivist/merkletree/archivist/archivist.nim b/archivist/merkletree/archivist/archivist.nim index 9efd4cbf..30c72e04 100644 --- a/archivist/merkletree/archivist/archivist.nim +++ b/archivist/merkletree/archivist/archivist.nim @@ -47,28 +47,6 @@ type ArchivistProof* = ref object of ByteProof mcodec*: MultiCodec -# CodeHashes is not exported from libp2p -# So we need to recreate it instead of -proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} = - for item in HashesList: - result[item.mcodec] = item - -const CodeHashes = initMultiHashCodeTable() - -func mhash*(mcodec: MultiCodec): ?!MHash = - let mhash = CodeHashes.getOrDefault(mcodec) - - if isNil(mhash.coder): - return failure "Invalid multihash codec" - - success mhash - -func digestSize*(self: (ArchivistTree or ArchivistProof)): int = - ## Number of leaves - ## - - self.mhash.size - func getProof*(self: ArchivistTree, index: int): ?!ArchivistProof = var proof = ArchivistProof(mcodec: self.mcodec) @@ -131,17 +109,12 @@ proc `$`*(self: ArchivistProof): string = ", path: " & $self.path.mapIt(byteutils.toHex(it)) & ", mcodec: " & $self.mcodec & " )" -func compress*(x, y: openArray[byte], key: ByteTreeKey, mhash: MHash): ?!ByteHash = +func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!ByteHash = ## Compress two hashes ## - - # Using Constantine's SHA256 instead of mhash for optimal performance on 32-byte merkle node hashing - # See: https://github.com/logos-storage/nim-codex/issues/1162 - let input = @x & @y & @[key.byte] - var digest = hashes.sha256.hash(input) - - success @digest + let digest = ?MultiHash.digest(codec, input).mapFailure + success digest.digestBytes func init*( _: type ArchivistTree, @@ -152,12 +125,12 @@ func init*( return failure "Empty leaves" let - mhash = ?mcodec.mhash() compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = - compress(x, y, key, mhash) - Zero: ByteHash = newSeq[byte](mhash.size) + compress(x, y, key, mcodec) + digestSize = ?mcodec.digestSize.mapFailure + Zero: ByteHash = newSeq[byte](digestSize) - if mhash.size != leaves[0].len: + if digestSize != leaves[0].len: return failure "Invalid hash length" var self = ArchivistTree(mcodec: mcodec, compress: compressor, zero: Zero) @@ -195,12 +168,12 @@ proc fromNodes*( return failure "Empty nodes" let - mhash = ?mcodec.mhash() - Zero = newSeq[byte](mhash.size) + digestSize = ?mcodec.digestSize.mapFailure + Zero = newSeq[byte](digestSize) compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = - compress(x, y, key, mhash) + compress(x, y, key, mcodec) - if mhash.size != nodes[0].len: + if digestSize != nodes[0].len: return failure "Invalid hash length" var @@ -233,10 +206,10 @@ func init*( return failure "Empty nodes" let - mhash = ?mcodec.mhash() - Zero = newSeq[byte](mhash.size) + digestSize = ?mcodec.digestSize.mapFailure + Zero = newSeq[byte](digestSize) compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} = - compress(x, y, key, mhash) + compress(x, y, key, mcodec) success ArchivistProof( compress: compressor, diff --git a/nim.cfg b/nim.cfg index 2b01522b..062d67bd 100644 --- a/nim.cfg +++ b/nim.cfg @@ -4,6 +4,11 @@ # archivist-dht only supports secp256k1 keys for libp2p nodes --define:"libp2p_pki_schemes=secp256k1" +# libp2p extensions for poseidon2 hashes +--define:"libp2p_multicodec_exts:../../../../archivist/libp2p/multicodec_exts.nim" +--define:"libp2p_multihash_exts:../../../../archivist/libp2p/multihash_exts.nim" +--define:"libp2p_contentids_exts:../../../../archivist/libp2p/contentids_exts.nim" + # confutils needs old-style case objects --define:"nimOldCaseObjects" diff --git a/tests/archivist/merkletree/testarchivisttree.nim b/tests/archivist/merkletree/testarchivisttree.nim index e8b4157f..230245be 100644 --- a/tests/archivist/merkletree/testarchivisttree.nim +++ b/tests/archivist/merkletree/testarchivisttree.nim @@ -82,10 +82,10 @@ suite "Test ArchivistTree": tree == fromNodes let - mhash = sha256.mhash().tryGet - zero: seq[byte] = newSeq[byte](mhash.size) + digestSize = sha256.digestSize.get + zero: seq[byte] = newSeq[byte](digestSize) compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] = - compress(x, y, key, mhash).tryGet + compress(x, y, key, sha256).tryGet makeTree = proc(data: seq[seq[byte]]): ArchivistTree = ArchivistTree.init(sha256, leaves = data).tryGet diff --git a/vendor/nimble/chronos b/vendor/nimble/chronos index 0646c444..0d00279e 160000 --- a/vendor/nimble/chronos +++ b/vendor/nimble/chronos @@ -1 +1 @@ -Subproject commit 0646c444fce7c7ed08ef6f2c9a7abfd172ffe655 +Subproject commit 0d00279e67ad9fadeb944944449adc89f052b8bd diff --git a/vendor/nimble/confutils b/vendor/nimble/confutils index 20ae94b4..f684e55d 160000 --- a/vendor/nimble/confutils +++ b/vendor/nimble/confutils @@ -1 +1 @@ -Subproject commit 20ae94b46852e99d9d548ca1af9e7cc1777b1272 +Subproject commit f684e55d56ba4016e2add64f74c4840476aa493d diff --git a/vendor/nimble/contractabi b/vendor/nimble/contractabi index e9092906..4f6eed81 160000 --- a/vendor/nimble/contractabi +++ b/vendor/nimble/contractabi @@ -1 +1 @@ -Subproject commit e9092906a5b10de1f83e48c6e8b92ac5acfa3a6e +Subproject commit 4f6eed812c5d84c5c1f9f966d448e026ee4c0d45 diff --git a/vendor/nimble/eth b/vendor/nimble/eth index dcfbc429..d9135e6c 160000 --- a/vendor/nimble/eth +++ b/vendor/nimble/eth @@ -1 +1 @@ -Subproject commit dcfbc4291d39b59563828c3e32be4d51a2f25931 +Subproject commit d9135e6c3c5d6d819afdfb566aa8d958756b73a8 diff --git a/vendor/nimble/ethers b/vendor/nimble/ethers index f547bd7c..67ccb347 160000 --- a/vendor/nimble/ethers +++ b/vendor/nimble/ethers @@ -1 +1 @@ -Subproject commit f547bd7cf23c7517af9188c082fb953a6cc7b71e +Subproject commit 67ccb347715f53b413af261e3e6d4d3e15421ffb diff --git a/vendor/nimble/httputils b/vendor/nimble/httputils index 79cbab14..c53852d9 160000 --- a/vendor/nimble/httputils +++ b/vendor/nimble/httputils @@ -1 +1 @@ -Subproject commit 79cbab1460f4c0cdde2084589d017c43a3d7b4f1 +Subproject commit c53852d9e24205b6363bba517fa8ee7bde823691 diff --git a/vendor/nimble/json_rpc b/vendor/nimble/json_rpc index 9665c265..09b5e047 160000 --- a/vendor/nimble/json_rpc +++ b/vendor/nimble/json_rpc @@ -1 +1 @@ -Subproject commit 9665c265035f49f5ff94bbffdeadde68e19d6221 +Subproject commit 09b5e047ff121d750d7d3e3c04847d8412887164 diff --git a/vendor/nimble/json_serialization b/vendor/nimble/json_serialization index e076cb9a..c343b0e2 160000 --- a/vendor/nimble/json_serialization +++ b/vendor/nimble/json_serialization @@ -1 +1 @@ -Subproject commit e076cb9a2dff30e13aad0bb4a13049da86d07967 +Subproject commit c343b0e243d9e17e2c40f3a8a24340f7c4a71d44 diff --git a/vendor/nimble/libp2p b/vendor/nimble/libp2p index 585021fb..a49f4b3f 160000 --- a/vendor/nimble/libp2p +++ b/vendor/nimble/libp2p @@ -1 +1 @@ -Subproject commit 585021fb92ae7f0080e43a9903b5ea18b5514094 +Subproject commit a49f4b3f9c60096f06f610311e9f43c0e9dfec3a diff --git a/vendor/nimble/lsquic b/vendor/nimble/lsquic new file mode 160000 index 00000000..4fb03ee7 --- /dev/null +++ b/vendor/nimble/lsquic @@ -0,0 +1 @@ +Subproject commit 4fb03ee7bfb39aecb3316889fdcb60bec3d0936f diff --git a/vendor/nimble/minilru b/vendor/nimble/minilru index aba86fcf..e3d6e6eb 160000 --- a/vendor/nimble/minilru +++ b/vendor/nimble/minilru @@ -1 +1 @@ -Subproject commit aba86fcf59597f5d43875751d478732100456d1f +Subproject commit e3d6e6eb5f119a0b22fdab418f7a588be462ea66 diff --git a/vendor/nimble/nimcrypto b/vendor/nimble/nimcrypto index 8085515e..b3dbc9c4 160000 --- a/vendor/nimble/nimcrypto +++ b/vendor/nimble/nimcrypto @@ -1 +1 @@ -Subproject commit 8085515e717b07e29de1ef50d5e9f15a3f6004c0 +Subproject commit b3dbc9c4d08e58c5b7bfad6dc7ef2ee52f2f4c08 diff --git a/vendor/nimble/secp256k1 b/vendor/nimble/secp256k1 index 9dd3df62..d8f1288b 160000 --- a/vendor/nimble/secp256k1 +++ b/vendor/nimble/secp256k1 @@ -1 +1 @@ -Subproject commit 9dd3df62124aae79d564da636bb22627c53c7676 +Subproject commit d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15 diff --git a/vendor/nimble/serialization b/vendor/nimble/serialization index b5193007..b0f2fa32 160000 --- a/vendor/nimble/serialization +++ b/vendor/nimble/serialization @@ -1 +1 @@ -Subproject commit b5193007a49639b21b2b80cd8ef8cbe0df6b0e48 +Subproject commit b0f2fa32960ea532a184394b0f27be37bd80248b diff --git a/vendor/nimble/snappy b/vendor/nimble/snappy index 8ab5cf0b..00bfcef9 160000 --- a/vendor/nimble/snappy +++ b/vendor/nimble/snappy @@ -1 +1 @@ -Subproject commit 8ab5cf0bdd84c7166b64dbe1b2deaa606d46ad22 +Subproject commit 00bfcef94f8ef6981df5d5b994897f6695badfb2 diff --git a/vendor/nimble/sqlite3_abi b/vendor/nimble/sqlite3_abi index a322ceb6..89ba51f5 160000 --- a/vendor/nimble/sqlite3_abi +++ b/vendor/nimble/sqlite3_abi @@ -1 +1 @@ -Subproject commit a322ceb619f40a5c521ea711ee1fad4c6d11aa09 +Subproject commit 89ba51f557414d3a3e17ab3df8270e1bdaa3ca2a diff --git a/vendor/nimble/stint b/vendor/nimble/stint index 7f83a754..baf249c5 160000 --- a/vendor/nimble/stint +++ b/vendor/nimble/stint @@ -1 +1 @@ -Subproject commit 7f83a754d5c72f927baff5cdd79013666d389ff7 +Subproject commit baf249c5cbe2aec7dc766753f3b68f364d1e946a diff --git a/vendor/nimble/zlib b/vendor/nimble/zlib index c71efff5..e680f269 160000 --- a/vendor/nimble/zlib +++ b/vendor/nimble/zlib @@ -1 +1 @@ -Subproject commit c71efff5fd1721362b3363dc7d0e2a4c0dbc6453 +Subproject commit e680f269fb01af2c34a2ba879ff281795a5258fe