From 8a5bfbac2deb7290734feaea83f8b92334efc0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ha=C5=82as?= Date: Tue, 8 Sep 2026 23:36:05 +0200 Subject: [PATCH] feat(#91): add ReadTransaction::getMappedRange() for single-round-trip index lookups --- CHANGELOG.md | 14 ++ docker/php/Dockerfile | 8 +- docs/range-reads.md | 63 +++++++ src/Future/FutureMappedKeyValueArray.php | 103 +++++++++++ src/Future/MappedRangeResult.php | 23 +++ src/MappedKeyValue.php | 25 +++ src/NativeClient.php | 69 ++++++++ src/ReadTransaction.php | 75 ++++++++ tests/Integration/MappedRangeTest.php | 213 +++++++++++++++++++++++ 9 files changed, 592 insertions(+), 1 deletion(-) create mode 100644 src/Future/FutureMappedKeyValueArray.php create mode 100644 src/Future/MappedRangeResult.php create mode 100644 src/MappedKeyValue.php create mode 100644 tests/Integration/MappedRangeTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cdebe8..27d6a84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,20 @@ ## [Unreleased] ### Added +- [#91] `ReadTransaction::getMappedRange()` — single-round-trip index lookups + backed by `fdb_transaction_get_mapped_range` and + `fdb_future_get_mappedkeyvalue_array` (the latter through the new + `CrazyGoat\FoundationDB\Future\FutureMappedKeyValueArray` future type). + Each resolved index row is returned as a `MappedKeyValue` pairing the index + key/value with the list of records described by the mapper tuple template + (`{K[N]}`, `{V[N]}`, and the `{...}` range descriptor that must be the last + element). Only available on non-snapshot (read-your-writes) reads — calling + it on a `Snapshot` throws a `LogicException`. The FFI layer also corrects + the `FDBMappedKeyValue` memory layout: the native reply stores either a + point lookup or a range lookup per row (selected by a variant index at + offset 104), which the public `fdb_c.h` declaration does not model. + Integration tests in `tests/Integration/MappedRangeTest.php`; + `docs/range-reads.md` updated with the mapper syntax. - [#90] `ReadTransaction::getTotalCost()` and `ReadTransaction::getTagThrottledDuration()` (backed by `fdb_transaction_get_total_cost` and diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index fe2ffc4..635b957 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -1,6 +1,7 @@ FROM php:8.2-cli-bookworm ARG FDB_VERSION=7.3.75 +ARG TARGETARCH RUN apt-get update && apt-get install -y --no-install-recommends \ wget \ @@ -11,7 +12,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ procps \ && rm -rf /var/lib/apt/lists/* -RUN wget -q "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" \ +# Map Docker's TARGETARCH (arm64/amd64) to the FoundationDB deb suffix (aarch64/amd64). +RUN case "$TARGETARCH" in \ + arm64) FDB_ARCH=aarch64 ;; \ + *) FDB_ARCH=amd64 ;; \ + esac \ + && wget -q "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb" \ -O /tmp/fdb-clients.deb \ && dpkg -i /tmp/fdb-clients.deb \ && rm /tmp/fdb-clients.deb diff --git a/docs/range-reads.md b/docs/range-reads.md index a016f18..f33b82a 100644 --- a/docs/range-reads.md +++ b/docs/range-reads.md @@ -127,6 +127,69 @@ $results = $db->getRangeAllStartsWith('users/'); // list Note: Database-level methods always return arrays (not lazy iterators). +## Mapped Range Reads (single-round-trip index lookups) + +`getMappedRange()` performs an index lookup and the corresponding record +fetches in a single round trip to the storage servers (backed by +`fdb_transaction_get_mapped_range`). Without it, an index-backed read requires +a `getRange()` over the index followed by N separate `get()` calls; with it, +every index row comes back together with its records in one response. + +The index and the records must be laid out so that the mapper can address the +records from an index row. The mapper is a **packed tuple template** whose +elements are resolved per index row: + +| Element | Resolves to | +|--------------|----------------------------------------------------------| +| `'literal'` | Copied verbatim into the record key/value | +| `{K[N]}` | The N-th element (0-based) of the index key tuple | +| `{V[N]}` | The N-th element (0-based) of the index value tuple | +| `{...}` | Range descriptor — must be the **last** element | + +- Without `{...}`, the mapper describes a **point lookup**: exactly one + record (the key built from the template) is fetched per index row. A + missing record yields an empty `results` list. +- With `{...}` as the last element, the mapper describes a **range lookup**: + every record whose key has the built key as a prefix is fetched per index + row. +- Literal braces can be escaped by doubling them (`{{`, `}}`). + +Example layout — index `('user', $userId) -> $name`, records under +`('records', $userId)` (multiple keys per user): + +```php +use CrazyGoat\FoundationDB\Tuple\Tuple; + +$tr = $db->createTransaction(); +[$begin, $end] = Tuple::range(['user']); // index range + +// mapper: ('records', {K[1]}, {...}) — every record under the +// ('records', $userId) prefix is fetched together with its index row +$mapper = Tuple::pack(['records', '{K[1]}', '{...}']); + +$result = $tr->getMappedRange($begin, $end, $mapper)->await(); + +foreach ($result->entries as $mapped) { + // $mapped->key / $mapped->value — the index row + // $mapped->results — list fetched for this row +} + +// Pagination: $result->more tells whether more index rows exist, +// $result->count the number of rows in this batch. Options (limit, +// reverse, streaming mode) are passed as a RangeOptions. +``` + +Notes: + +- Mapped ranges are **not** supported on snapshot reads on current + FoundationDB versions — call `getMappedRange()` on a plain `Transaction` + (read-your-writes). Calling it on `Transaction::snapshot()` throws a + `LogicException`. +- An invalid mapper (unparseable tuple, bad placeholder, or a `{...}` + descriptor that is not the last element) surfaces as an `FDBException`. +- `getRange()`-style lazy pagination is not wrapped for mapped reads; use the + `RangeOptions` limit together with `more` to page manually. + ## Range Size Estimation ```php diff --git a/src/Future/FutureMappedKeyValueArray.php b/src/Future/FutureMappedKeyValueArray.php new file mode 100644 index 0000000..835ad5f --- /dev/null +++ b/src/Future/FutureMappedKeyValueArray.php @@ -0,0 +1,103 @@ +resolved && $this->cachedResult instanceof MappedRangeResult) { + return $this->cachedResult; + } + + $this->blockUntilReady(); + + $outKvm = $this->client->fdb->new('FDBMappedKeyValue*'); + $outCount = $this->client->fdb->new('int'); + $outMore = $this->client->fdb->new('fdb_bool_t'); + + $this->client->checkError( + $this->client->fdb->fdb_future_get_mappedkeyvalue_array( + $this->fpointer, + FFI::addr($outKvm), + FFI::addr($outCount), + FFI::addr($outMore), + ), + ); + + $count = $outCount->cdata; + $entries = []; + + for ($i = 0; $i < $count; $i++) { + $mapped = $outKvm[$i]; + + $indexKey = $this->stringFromKey($mapped->key); + $indexValue = $this->stringFromKey($mapped->value); + + $variant = $mapped->variant_index; + + if ($variant === self::VARIANT_GET_VALUE) { + $results = []; + if ($mapped->reqAndResult->getValue->present) { + $results[] = new KeyValue( + $this->stringFromKey($mapped->reqAndResult->getValue->key), + $this->stringFromKey($mapped->reqAndResult->getValue->value), + ); + } + } elseif ($variant === self::VARIANT_GET_RANGE) { + $results = []; + $reqAndResult = $mapped->reqAndResult->getRange->reqAndResult; + $resultCount = $reqAndResult->m_size; + for ($j = 0; $j < $resultCount; $j++) { + $kv = $reqAndResult->data[$j]; + $results[] = new KeyValue( + FFI::string($kv->key, $kv->key_length), + FFI::string($kv->value, $kv->value_length), + ); + } + } else { + throw new \RuntimeException(sprintf( + 'Unknown mapped key value variant index: %d', + $variant, + )); + } + + $entries[] = new MappedKeyValue($indexKey, $indexValue, $results); + } + + $this->cachedResult = new MappedRangeResult($entries, $count, $outMore->cdata !== 0); + $this->releaseMemory(); + $this->resolved = true; + + return $this->cachedResult; + } + + private function stringFromKey(FFI\CData $key): string + { + return FFI::string($key->key, $key->key_length); + } +} diff --git a/src/Future/MappedRangeResult.php b/src/Future/MappedRangeResult.php new file mode 100644 index 0000000..cff9070 --- /dev/null +++ b/src/Future/MappedRangeResult.php @@ -0,0 +1,23 @@ + $entries + * @param int<0, max> $count + */ + public function __construct( + public array $entries, + public int $count, + public bool $more, + ) { + } +} diff --git a/src/MappedKeyValue.php b/src/MappedKeyValue.php new file mode 100644 index 0000000..38c8257 --- /dev/null +++ b/src/MappedKeyValue.php @@ -0,0 +1,25 @@ + $results The records mapped from this index row. + */ + public function __construct( + public string $key, + public string $value, + public array $results, + ) { + } +} diff --git a/src/NativeClient.php b/src/NativeClient.php index 2e0a32e..8191aa6 100644 --- a/src/NativeClient.php +++ b/src/NativeClient.php @@ -29,6 +29,62 @@ final class NativeClient int key_length; } FDBKey; + /* Memory layout of KeySelectorRef (packed via #pragma pack(4) in fdb_c.h). */ + typedef struct __attribute__((packed)) { + FDBKey key; + fdb_bool_t orEqual; + int offset; + } FDBKeySelector; + + /* Memory layout of GetRangeReqAndResultRef (packed via #pragma pack(4) in fdb_c.h). */ + typedef struct __attribute__((packed)) { + FDBKeySelector begin; + FDBKeySelector end; + FDBKeyValue* data; + int m_size; + int m_capacity; + } FDBGetRangeReqAndResult; + + /* Memory layout of MappedKeyValueRef (packed via #pragma pack(4) in fdb_c.h). */ + /* Memory layout of MappedKeyValueRef (not packed in fdb_c.h). + * + * The public fdb_c.h FDBMappedKeyValue only models the getRange + * alternative of the underlying std::variant, which is not what the + * native client actually produces: the reply carries either a point + * lookup (GetValueReqAndResultRef, variant index 0) or a range + * lookup (GetRangeReqAndResultRef, variant index 1). The real C++ + * object is laid out as: index key (12B), index value (12B), an + * 80-byte variant union at offset 24, and a 4-byte variant index at + * offset 104 (padded to a 112-byte stride). FDBGetValue / + * FDBGetRangeReqAndResultFull below mirror both alternatives over + * the 80-byte union, so the struct declared here mirrors the real + * memory layout rather than the one from fdb_c.h. + */ + typedef struct __attribute__((packed)) { + FDBKey key; + FDBKey value; + bool present; + unsigned char tail[55]; + } FDBGetValueReqAndResult; + + typedef struct __attribute__((packed)) { + FDBGetRangeReqAndResult reqAndResult; + unsigned char tail[24]; + } FDBGetRangeReqAndResultFull; + + typedef union __attribute__((packed)) { + FDBGetValueReqAndResult getValue; + FDBGetRangeReqAndResultFull getRange; + } FDBMappedReqAndResult; + + typedef struct __attribute__((packed)) { + FDBKey key; + FDBKey value; + FDBMappedReqAndResult reqAndResult; + int variant_index; + unsigned char tail[4]; + } FDBMappedKeyValue; + fdb_error_t fdb_select_api_version_impl(int runtime_version, int header_version); int fdb_get_max_api_version(); const char* fdb_get_error(fdb_error_t code); @@ -102,6 +158,19 @@ final class NativeClient int limit, int target_bytes, int streaming_mode, int iteration, fdb_bool_t snapshot, fdb_bool_t reverse ); + FDBFuture* fdb_transaction_get_mapped_range( + FDBTransaction* tr, + const char* begin_key_name, int begin_key_name_length, + fdb_bool_t begin_or_equal, int begin_offset, + const char* end_key_name, int end_key_name_length, + fdb_bool_t end_or_equal, int end_offset, + const char* mapper_name, int mapper_name_length, + int limit, int target_bytes, int streaming_mode, int iteration, + fdb_bool_t snapshot, fdb_bool_t reverse + ); + fdb_error_t fdb_future_get_mappedkeyvalue_array( + FDBFuture* f, const FDBMappedKeyValue** out_kvm, int* out_count, fdb_bool_t* out_more + ); FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, const char* begin_key_name, int begin_key_name_length, diff --git a/src/ReadTransaction.php b/src/ReadTransaction.php index 161820b..b605c0e 100644 --- a/src/ReadTransaction.php +++ b/src/ReadTransaction.php @@ -9,6 +9,7 @@ use CrazyGoat\FoundationDB\Future\FutureInt64; use CrazyGoat\FoundationDB\Future\FutureKey; use CrazyGoat\FoundationDB\Future\FutureKeyArray; +use CrazyGoat\FoundationDB\Future\FutureMappedKeyValueArray; use CrazyGoat\FoundationDB\Future\FutureStringArray; use CrazyGoat\FoundationDB\Future\FutureValue; use FFI\CData; @@ -168,6 +169,80 @@ public function getRange( ); } + /** + * Single-round-trip index lookups (fdb_transaction_get_mapped_range). + * + * Performs a range read over an index subspace and, for every index row, + * fetches the records described by the mapper — all in one round trip to + * the storage servers. Each resolved index row is returned as a + * MappedKeyValue pairing the index key/value with its records. + * + * The mapper is a packed tuple template (see docs/range-reads.md): + * elements like "{K[1]}" and "{V[0]}" are substituted with the + * corresponding tuple components of the index key/value, literal + * elements are copied verbatim, and "{...}" stands for the remaining + * tuple elements. + * + * NOTE: current FoundationDB versions only support mapped ranges on + * non-snapshot (read-your-writes) reads — call this on a Transaction. + * On a Snapshot read a LogicException is thrown; the native client also + * rejects the request with an FDBException. + */ + public function getMappedRange( + string|KeySelector $begin, + string|KeySelector $end, + string $mapper, + ?RangeOptions $options = null, + ): FutureMappedKeyValueArray { + if ($this->isSnapshot) { + throw new \LogicException( + 'getMappedRange() is only supported on non-snapshot (read-your-writes) reads; ' . + 'call it on a Transaction instead of Transaction::snapshot()', + ); + } + + $options ??= new RangeOptions(); + + $beginSelector = $begin instanceof KeySelector + ? $begin + : KeySelector::firstGreaterOrEqual($begin); + + $endSelector = $end instanceof KeySelector + ? $end + : KeySelector::firstGreaterOrEqual($end); + + // Validate the resulting range endpoints and the mapper eagerly so an + // oversize key fails at the call site instead of when awaiting. + $beginKeyLength = KeyValueLimits::assertValidRangeEndpoint($beginSelector->key); + $endKeyLength = KeyValueLimits::assertValidRangeEndpoint($endSelector->key); + $mapperLength = KeyValueLimits::assertValidFfiLength($mapper, 'Mapper template'); + + return new FutureMappedKeyValueArray( + $this->client->fdb->fdb_transaction_get_mapped_range( + $this->tpointer, + $beginSelector->key, + $beginKeyLength, + $beginSelector->orEqual ? 1 : 0, + $beginSelector->offset, + $endSelector->key, + $endKeyLength, + $endSelector->orEqual ? 1 : 0, + $endSelector->offset, + $mapper, + $mapperLength, + $options->limit ?? 0, + 0, + $options->mode->value, + 1, + // Mapped ranges are only supported on non-snapshot reads + // (see the guard above). + 0, + $options->reverse ? 1 : 0, + ), + $this->client, + ); + } + /** * @return list */ diff --git a/tests/Integration/MappedRangeTest.php b/tests/Integration/MappedRangeTest.php new file mode 100644 index 0000000..61c5e36 --- /dev/null +++ b/tests/Integration/MappedRangeTest.php @@ -0,0 +1,213 @@ +seedIndexAndRecords(); + + $tr = $this->getDatabase()->createTransaction(); + [$begin, $end] = Tuple::range([self::INDEX_SUBSPACE]); + + // Point mapper: ('mapped_records', {K[1]}) — a single record per row. + $result = $tr->getMappedRange($begin, $end, $this->pointMapper())->await(); + + self::assertSame(3, $result->count); + self::assertFalse($result->more); + + self::assertInstanceOf(MappedKeyValue::class, $result->entries[0]); + self::assertSame( + Tuple::pack([self::INDEX_SUBSPACE, 'alice']), + $result->entries[0]->key, + ); + self::assertSame('Alice', $result->entries[0]->value); + self::assertCount(1, $result->entries[0]->results); + self::assertSame( + Tuple::pack([self::RECORDS_SUBSPACE, 'alice']), + $result->entries[0]->results[0]->key, + ); + self::assertSame('record-alice', $result->entries[0]->results[0]->value); + } + + #[Test] + public function getMappedRangeWithRangeMapperReturnsAllMappedRecords(): void + { + $this->seedIndexAndRecords(); + + $tr = $this->getDatabase()->createTransaction(); + [$begin, $end] = Tuple::range([self::INDEX_SUBSPACE]); + + // Range mapper: ('mapped_records', {K[1]}, {...}) — every record + // under the ('mapped_records', id) prefix, fetched in the same + // round trip as the index rows. + $result = $tr->getMappedRange($begin, $end, $this->rangeMapper(), new RangeOptions( + mode: StreamingMode::WantAll, + ))->await(); + + self::assertSame(3, $result->count); + + foreach ($result->entries as $mapped) { + $id = $this->indexIdOf($mapped->key); + self::assertCount(3, $mapped->results); + self::assertSame( + Tuple::pack([self::RECORDS_SUBSPACE, $id]), + $mapped->results[0]->key, + ); + self::assertSame('record-' . $id, $mapped->results[0]->value); + self::assertSame( + Tuple::pack([self::RECORDS_SUBSPACE, $id, 'name']), + $mapped->results[1]->key, + ); + self::assertSame('name-' . $id, $mapped->results[1]->value); + self::assertSame( + Tuple::pack([self::RECORDS_SUBSPACE, $id, 'score']), + $mapped->results[2]->key, + ); + self::assertSame('score-' . $id, $mapped->results[2]->value); + } + } + + #[Test] + public function getMappedRangeResolvesValueComponents(): void + { + // The record key is built from the index VALUE instead of the key: + // index ('mapped_index', $id) -> $id, mapper ('mapped_records', {V[0]}). + $this->getDatabase()->transact(function (Transaction $tr): void { + foreach (['x', 'y'] as $id) { + $tr->set(Tuple::pack([self::INDEX_SUBSPACE, $id]), Tuple::pack([$id])); + $tr->set(Tuple::pack([self::RECORDS_SUBSPACE, $id]), 'rec-' . $id); + } + }); + + $tr = $this->getDatabase()->createTransaction(); + [$begin, $end] = Tuple::range([self::INDEX_SUBSPACE]); + $mapper = Tuple::pack([self::RECORDS_SUBSPACE, '{V[0]}']); + + $result = $tr->getMappedRange($begin, $end, $mapper)->await(); + + self::assertCount(2, $result->entries); + + foreach ($result->entries as $mapped) { + $unpacked = Tuple::unpack($mapped->value); + $id = is_string($unpacked[0] ?? null) ? $unpacked[0] : ''; + self::assertSame($id, $this->indexIdOf($mapped->key)); + self::assertCount(1, $mapped->results); + self::assertSame(Tuple::pack([self::RECORDS_SUBSPACE, $id]), $mapped->results[0]->key); + self::assertSame('rec-' . $id, $mapped->results[0]->value); + } + } + + #[Test] + public function getMappedRangeRespectsLimit(): void + { + $this->seedIndexAndRecords(); + + $tr = $this->getDatabase()->createTransaction(); + [$begin, $end] = Tuple::range([self::INDEX_SUBSPACE]); + + $result = $tr->getMappedRange($begin, $end, $this->pointMapper(), new RangeOptions( + limit: 2, + mode: StreamingMode::WantAll, + ))->await(); + + self::assertSame(2, $result->count); + self::assertCount(2, $result->entries); + } + + #[Test] + public function getMappedRangeOnEmptyRange(): void + { + $tr = $this->getDatabase()->createTransaction(); + [$begin, $end] = Tuple::range([self::INDEX_SUBSPACE]); + + $result = $tr->getMappedRange($begin, $end, $this->pointMapper())->await(); + + self::assertSame(0, $result->count); + self::assertSame([], $result->entries); + } + + #[Test] + public function getMappedRangeRejectsSnapshotReads(): void + { + $this->seedIndexAndRecords(); + + $snap = $this->getDatabase()->createTransaction()->snapshot(); + [$begin, $end] = Tuple::range([self::INDEX_SUBSPACE]); + + $this->expectException(\LogicException::class); + $snap->getMappedRange($begin, $end, $this->pointMapper()); + } + + #[Test] + public function getMappedRangeWithInvalidMapperThrowsFdbException(): void + { + $this->seedIndexAndRecords(); + + $tr = $this->getDatabase()->createTransaction(); + [$begin, $end] = Tuple::range([self::INDEX_SUBSPACE]); + + // A mapper that is not a parseable tuple template. + $future = $tr->getMappedRange($begin, $end, "not-a-tuple\xFF\xFF\xFF"); + + $this->expectException(FDBException::class); + $future->await(); + } + + /** + * Mapper: ('mapped_records', {K[1]}) — a point lookup of the second + * component of the index key tuple. + */ + private function pointMapper(): string + { + return Tuple::pack([self::RECORDS_SUBSPACE, '{K[1]}']); + } + + /** + * Mapper: ('mapped_records', {K[1]}, {...}) — a range lookup over the + * ('mapped_records', id) prefix. The "{...}" descriptor must be the + * last element of the mapper tuple. + */ + private function rangeMapper(): string + { + return Tuple::pack([self::RECORDS_SUBSPACE, '{K[1]}', '{...}']); + } + + private function indexIdOf(string $indexKey): string + { + $unpacked = Tuple::unpack($indexKey); + + return is_string($unpacked[1] ?? null) ? $unpacked[1] : ''; + } + + private function seedIndexAndRecords(): void + { + $this->getDatabase()->transact(function (Transaction $tr): void { + foreach (['alice', 'bob', 'carol'] as $id) { + $tr->set(Tuple::pack([self::INDEX_SUBSPACE, $id]), ucfirst($id)); + $tr->set(Tuple::pack([self::RECORDS_SUBSPACE, $id]), 'record-' . $id); + $tr->set(Tuple::pack([self::RECORDS_SUBSPACE, $id, 'name']), 'name-' . $id); + $tr->set(Tuple::pack([self::RECORDS_SUBSPACE, $id, 'score']), 'score-' . $id); + } + }); + } +}