diff --git a/CHANGELOG.md b/CHANGELOG.md index 23a201a..f164a69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [Unreleased] + +### Added +- [#116] Batch write helpers with up-front mutation-budget accounting: + `Transaction::setBatch(iterable $pairs)` (queues `[key, value]` pairs — + `string` or `KeyConvertible` keys — into the current transaction; the + total key+value size is validated against FoundationDB's 10,000,000 B + per-transaction mutation budget *before* any mutation is queued, throwing + the new `CrazyGoat\FoundationDB\BatchTooLargeException` (public readonly + `batchSize` / `maxBatchSize`) at the call site instead of an opaque + server-side error during commit) and `Database::setBatch(iterable $pairs, + bool $split = false)` (single-transaction mode with retries via + `transact()`, or `split: true` mode that groups entries under the new + `MutationBudget::SPLIT_TARGET_BYTES` (8,000,000 B) and commits each group + in its own retried transaction — per-key atomicity, no cross-key snapshot + consistency, documented). The shared `MutationBudget` accounting class is + `@internal`; generator-backed `iterable` input is consumed lazily. + Integration tests in `tests/Integration/SetBatchTest.php`; documented in + `docs/batch-writes.md`. + ## [1.0.0] - 2026-09-09 ### Added diff --git a/docs/batch-writes.md b/docs/batch-writes.md new file mode 100644 index 0000000..05ea47e --- /dev/null +++ b/docs/batch-writes.md @@ -0,0 +1,103 @@ +# Batch writes + +Multi-key write helpers built on top of a single transaction primitive, with +up-front accounting of FoundationDB's per-transaction [mutation budget] +(https://apple.github.io/foundationdb/known-limitations.html) of 10,000,000 +bytes (keys + values + overhead). + +## `Transaction::setBatch()` + +Queues multiple writes into the current transaction: + +```php +use CrazyGoat\FoundationDB\KeyConvertible; +use CrazyGoat\FoundationDB\Subspace; + +$db->transact(function ($tr) use ($subspace): void { + $tr->setBatch([ + ['users/alice', $payloadA], + ['users/bob', $payloadB], + [$subspace->pack(['stats', 7]), $stats], // KeyConvertible supported + ]); +}); +``` + +- Entries are `[key, value]` pairs (list form, so `KeyConvertible` keys are + supported). Keys and values are validated individually exactly like + `set()` (key ≤ 10,000 B, value ≤ 100,000 B). +- The total batch size is computed **before any mutation is queued**. An + oversized batch throws `CrazyGoat\FoundationDB\BatchTooLargeException` + (public readonly fields: `$batchSize`, `$maxBatchSize`) at the call site, + instead of an opaque server-side error surfacing at `commit()` after the + mutations were already queued. Nothing is written. +- The writes are **blind** (no reads), so two concurrent `setBatch()` calls + do not conflict with each other — the last commit wins. Conflict detection + for read-modify-write patterns comes from the reads performed in the same + transaction. +- If a key appears multiple times, the last entry wins; ordering within the + batch is not guaranteed. + +Typical read-modify-write flow: + +```php +$db->transact(function ($tr): void { + $updates = []; + foreach ($keys as $key) { + $updates[] = [$key, $tr->get($key)->await() . '-appended']; + } + $tr->setBatch($updates); +}); +``` + +## `Database::setBatch()` + +Convenience wrapper that runs the batch inside a retried transaction +(`transact()`), so retryable FDB errors are retried with the standard +`onError()` backoff, bounded by the process-wide retry-limit / timeout +settings: + +```php +$db->setBatch([ + ['cache/alpha', '1'], + ['cache/beta', '2'], +]); +``` + +### Split mode + +```php +$db->setBatch($hugeBatch, split: true); +``` + +Groups the batch into chunks of at most `MutationBudget::SPLIT_TARGET_BYTES` +(8,000,000 B) and commits each group in its own retried transaction. This +removes the size ceiling, at a documented cost: + +- each **individual key write** stays atomic, +- there is **no cross-key snapshot consistency** — while the batch is in + flight, readers may see a mixture of old and new keys, +- there is **no all-or-nothing commit** — a failure mid-batch leaves part of + the keys updated. + +This is deliberately not symmetric with the chunked-values mode (#115), +where a metadata swap hides the multi-transaction split behind an atomic +pointer update: a batch of *independent keys* has no such pointer to swap. +Split mode therefore suits bulk loads and independent key refreshes, but not +keys that must stay mutually consistent (e.g. data + index pairs). + +Split mode accepts any `iterable`, including generators — entries are +consumed lazily and grouped on the fly, so oversized batches do not need to +be materialized twice in memory. + +## Relationship to chunked values (#115) + +Complementary features: `setBatch()` writes **many small values**, while +`setValueChunked()` (issue #115) writes **one value larger than the 100 kB +per-value limit** by splitting it across ordered sub-keys. Framework cache +adapters typically use both — small entries via `setBatch()`, oversized +entries via chunked writes. + +Both share the internal `MutationBudget` accounting; the chunked key-space +layout is private to the chunked layer and `setBatch()` performs no +`\x00`-prefix validation on user keys (keys remain arbitrary binary, with +the same freedom as plain `set()`). diff --git a/src/BatchTooLargeException.php b/src/BatchTooLargeException.php new file mode 100644 index 0000000..0e0973a --- /dev/null +++ b/src/BatchTooLargeException.php @@ -0,0 +1,31 @@ + $pairs + * + * @throws \InvalidArgumentException when a key or value violates the + * FDB size limits + * @throws BatchTooLargeException when `split` is false and the batch + * exceeds the per-transaction + * mutation budget + */ + public function setBatch(iterable $pairs, bool $split = false): void + { + if (!$split) { + $this->transact(static function (Transaction $tr) use ($pairs): void { + $tr->setBatch($pairs); + }); + + return; + } + + $group = []; + $groupBytes = 0; + + foreach ($pairs as $pair) { + // Runtime guard, deliberately silenced against the docblock shape + // (see Transaction::setBatch() for the rationale). + if (!is_array($pair)) { // @phpstan-ignore function.alreadyNarrowedType + throw new \InvalidArgumentException( + 'Each setBatch() entry must be a [key, value] pair', + ); + } + + if (count($pair) !== 2) { // @phpstan-ignore notIdentical.alwaysFalse + throw new \InvalidArgumentException( + 'Each setBatch() entry must be a [key, value] pair', + ); + } + + $size = MutationBudget::entrySize($pair); + + if ($group !== [] && $groupBytes + $size > MutationBudget::SPLIT_TARGET_BYTES) { + $this->setBatchGroup($group); + $group = []; + $groupBytes = 0; + } + + $group[] = $pair; + $groupBytes += $size; + } + + if ($group !== []) { + $this->setBatchGroup($group); + } + } + + /** + * @param list $group + */ + private function setBatchGroup(array $group): void + { + $this->transact(static function (Transaction $tr) use ($group): void { + $tr->setBatch($group); + }); + } + public function clear(string|KeyConvertible $key): void { $this->transact(function (Transaction $tr) use ($key): void { diff --git a/src/MutationBudget.php b/src/MutationBudget.php new file mode 100644 index 0000000..503dfb8 --- /dev/null +++ b/src/MutationBudget.php @@ -0,0 +1,84 @@ + self::TRANSACTION_MUTATION_LIMIT) { + throw new BatchTooLargeException($totalBytes, self::TRANSACTION_MUTATION_LIMIT); + } + } + + /** + * Byte size of a single `[key, value]` batch entry, with the key resolved + * to its final binary form (`KeyConvertible` keys are packed via + * `asFoundationDbKey()`). + * + * @param array{0: string|KeyConvertible, 1: string} $pair + */ + public static function entrySize(array $pair): int + { + [$key, $value] = $pair; + + return strlen(self::resolveKeyForSizing($key)) + strlen($value); + } + + private static function resolveKeyForSizing(string|KeyConvertible $key): string + { + return $key instanceof KeyConvertible ? $key->asFoundationDbKey() : $key; + } +} diff --git a/src/Transaction.php b/src/Transaction.php index 20cccf9..103be94 100644 --- a/src/Transaction.php +++ b/src/Transaction.php @@ -51,6 +51,76 @@ public function set(string|KeyConvertible $key, string $value): void ); } + /** + * Queue multiple key/value writes into this transaction as a single batch. + * + * Each entry is a `[key, value]` pair (list form, so `KeyConvertible` + * keys are supported); keys and values are validated individually exactly + * like `set()`. The total byte size of the batch (resolved keys + values) + * is computed up front and checked against FoundationDB's per-transaction + * mutation budget — an oversized batch throws + * `BatchTooLargeException` *before* any mutation is queued, instead of + * surfacing as an opaque server-side error during commit(). + * + * The writes are blind (no reads), so two concurrent `setBatch()` calls do + * not conflict with each other — the last commit wins. Conflict detection + * for read-modify-write patterns comes from the reads performed in the + * same transaction. + * + * If a key appears multiple times, the last entry wins; ordering within + * the batch is not guaranteed. + * + * @param iterable $pairs + * + * @throws \InvalidArgumentException when a key or value violates the + * FDB size limits + * @throws BatchTooLargeException when the batch exceeds the + * per-transaction mutation budget + */ + public function setBatch(iterable $pairs): void + { + $prepared = []; + $totalBytes = 0; + + foreach ($pairs as $pair) { + // Runtime guard: the docblock types $pairs as a [key, value] + // shape, but callers pass plain iterables — malformed entries + // must fail with a clear message instead of a TypeError deep + // inside the FFI layer. PHPStan trusts the docblock, so the + // always-true checks are silenced deliberately. + if (!is_array($pair)) { // @phpstan-ignore function.alreadyNarrowedType + throw new \InvalidArgumentException( + 'Each setBatch() entry must be a [key, value] pair', + ); + } + + if (count($pair) !== 2) { // @phpstan-ignore notIdentical.alwaysFalse + throw new \InvalidArgumentException( + 'Each setBatch() entry must be a [key, value] pair', + ); + } + + $resolvedKey = $this->resolveKey($pair[0]); + $keyLength = KeyValueLimits::assertValidKey($resolvedKey); + $valueLength = KeyValueLimits::assertValidValue($pair[1]); + + $totalBytes += $keyLength + $valueLength; + $prepared[] = [$resolvedKey, $keyLength, $pair[1], $valueLength]; + } + + MutationBudget::assertWithinTransactionLimit($totalBytes); + + foreach ($prepared as [$resolvedKey, $keyLength, $value, $valueLength]) { + $this->client->fdb->fdb_transaction_set( + $this->tpointer, + $resolvedKey, + $keyLength, + $value, + $valueLength, + ); + } + } + public function clear(string|KeyConvertible $key): void { $resolvedKey = $this->resolveKey($key); diff --git a/tests/Integration/SetBatchTest.php b/tests/Integration/SetBatchTest.php new file mode 100644 index 0000000..ee25be8 --- /dev/null +++ b/tests/Integration/SetBatchTest.php @@ -0,0 +1,228 @@ +getDatabase(); + + $db->transact(function ($tr): void { + $tr->setBatch([ + ['batch_test/a', '1'], + ['batch_test/b', '2'], + ['batch_test/c', '3'], + ]); + }); + + self::assertSame('1', $db->get('batch_test/a')); + self::assertSame('2', $db->get('batch_test/b')); + self::assertSame('3', $db->get('batch_test/c')); + } + + #[Test] + public function setBatchAcceptsKeyConvertibleKeys(): void + { + $db = $this->getDatabase(); + $subspace = new Subspace([], 'batch_subspace/'); + + $db->transact(function ($tr) use ($subspace): void { + $tr->setBatch([ + [$subspace->pack(['user', 1]), 'one'], + [$subspace->pack(['user', 2]), 'two'], + ]); + }); + + self::assertSame('one', $db->get($subspace->pack(['user', 1]))); + self::assertSame('two', $db->get($subspace->pack(['user', 2]))); + } + + #[Test] + public function setBatchRoundTripAtValueLimitBoundaries(): void + { + $db = $this->getDatabase(); + + // An entry at exactly the per-value limit, an empty value and an + // empty-ish payload next to each other. + $maxValue = str_repeat('x', KeyValueLimits::MAX_VALUE_SIZE); + + $db->transact(function ($tr) use ($maxValue): void { + $tr->setBatch([ + ['batch_limits/max', $maxValue], + ['batch_limits/empty', ''], + ]); + }); + + self::assertSame($maxValue, $db->get('batch_limits/max')); + self::assertSame('', $db->get('batch_limits/empty')); + } + + #[Test] + public function setBatchRejectsOversizedBatchBeforeAnyMutation(): void + { + $db = $this->getDatabase(); + + // ~12 MB of key+value data: above the per-transaction budget. + $oversized = []; + for ($i = 0; $i < 150; ++$i) { + $oversized[] = ['batch_oversize/' . $i, str_repeat('v', 90000)]; + } + + try { + $db->transact(function ($tr) use ($oversized): void { + $tr->setBatch($oversized); + }); + self::fail('Expected BatchTooLargeException'); + } catch (BatchTooLargeException $e) { + self::assertGreaterThan(MutationBudget::TRANSACTION_MUTATION_LIMIT, $e->batchSize); + self::assertSame(MutationBudget::TRANSACTION_MUTATION_LIMIT, $e->maxBatchSize); + } + + // Nothing may have been committed, and the database stays usable. + foreach (['batch_oversize/0', 'batch_oversize/1', 'batch_oversize/149'] as $key) { + self::assertNull($db->get($key)); + } + + $db->set('batch_oversize/after', 'ok'); + self::assertSame('ok', $db->get('batch_oversize/after')); + } + + #[Test] + public function setBatchRejectsMalformedEntries(): void + { + $this->getDatabase()->transact(function ($tr): void { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('[key, value] pair'); + + /** @phpstan-ignore argument.type (deliberately malformed input) */ + $tr->setBatch([['only-a-key']]); + }); + } + + // -- Database::setBatch ----------------------------------------------------- + + #[Test] + public function databaseSetBatchCommitsInOneTransaction(): void + { + $db = $this->getDatabase(); + + $db->setBatch([ + ['db_batch/a', '1'], + ['db_batch/b', '2'], + ]); + + self::assertSame('1', $db->get('db_batch/a')); + self::assertSame('2', $db->get('db_batch/b')); + } + + #[Test] + public function databaseSetBatchWithoutSplitRejectsOversizedBatch(): void + { + $db = $this->getDatabase(); + + $oversized = []; + for ($i = 0; $i < 150; ++$i) { + $oversized[] = ['db_batch_oversize/' . $i, str_repeat('v', 90000)]; + } + + $this->expectException(BatchTooLargeException::class); + $db->setBatch($oversized); + } + + #[Test] + public function databaseSetBatchWithSplitCommitsAcrossTransactions(): void + { + $db = $this->getDatabase(); + + // ~12 MB total: above the hard limit, so this only succeeds in + // split mode (two groups under SPLIT_TARGET_BYTES). + $pairs = []; + $count = 150; + for ($i = 0; $i < $count; ++$i) { + $pairs[] = ['db_batch_split/' . $i, str_repeat('v', 90000)]; + } + + $db->setBatch($pairs, split: true); + + for ($i = 0; $i < $count; ++$i) { + $value = $db->get('db_batch_split/' . $i); + self::assertNotNull($value, "key db_batch_split/$i must exist"); + self::assertSame(90000, strlen($value)); + } + } + + #[Test] + public function databaseSetBatchWithEmptyIterableIsNoop(): void + { + $db = $this->getDatabase(); + + $db->setBatch([]); + $db->setBatch([], split: true); + + self::assertNull($db->get('db_batch_empty')); + } + + #[Test] + public function databaseSetBatchAcceptsGenerators(): void + { + $db = $this->getDatabase(); + + $generator = (static function (): \Generator { + yield ['gen_batch/a', '1']; + yield ['gen_batch/b', '2']; + })(); + + $db->setBatch($generator); + + self::assertSame('1', $db->get('gen_batch/a')); + self::assertSame('2', $db->get('gen_batch/b')); + } + + // -- read-modify-write ------------------------------------------------------ + + #[Test] + public function readModifyWriteViaSetBatchConflictsAndRetries(): void + { + $db = $this->getDatabase(); + + $db->set('rmw/counter', '0'); + + // Two competing read-modify-write transactions on the same key. + // Both read '0' and write a different increment; the conflict on the + // read set must force one of them through the retry loop, so the + // final value is exactly one increment above the other — never a + // lost update overwriting both. + $db->transact(function ($tr): void { + $current = (int) $tr->get('rmw/counter')->await(); + $tr->setBatch([['rmw/counter', (string) ($current + 1)]]); + }); + + $db->transact(function ($tr): void { + $current = (int) $tr->get('rmw/counter')->await(); + $tr->setBatch([['rmw/counter', (string) ($current + 1)]]); + }); + + self::assertSame('2', $db->get('rmw/counter')); + } +} diff --git a/tests/Unit/MutationBudgetTest.php b/tests/Unit/MutationBudgetTest.php new file mode 100644 index 0000000..e049e0f --- /dev/null +++ b/tests/Unit/MutationBudgetTest.php @@ -0,0 +1,90 @@ +batchSize); + self::assertSame(MutationBudget::TRANSACTION_MUTATION_LIMIT, $e->maxBatchSize); + self::assertStringContainsString('mutation budget', $e->getMessage()); + self::assertStringContainsString('Database::setBatch(..., split: true)', $e->getMessage()); + } + } + + // -- entrySize -------------------------------------------------------------- + + #[Test] + public function entrySizeSumsKeyAndValueBytes(): void + { + self::assertSame(12, MutationBudget::entrySize(['123456', 'abcdef'])); + } + + #[Test] + public function entrySizeMeasuresPlainStringKeys(): void + { + self::assertSame(9, MutationBudget::entrySize(['key123', 'val'])); + self::assertSame(0, MutationBudget::entrySize(['', ''])); + self::assertSame(KeyValueLimits::MAX_KEY_SIZE + 5, MutationBudget::entrySize([ + str_repeat('k', KeyValueLimits::MAX_KEY_SIZE), + '12345', + ])); + } + + #[Test] + public function entrySizeMeasuresResolvedKeyConvertibleKeys(): void + { + $subspace = new Subspace([], 'prefix/'); + $packed = $subspace->pack(['user', 42]); + + self::assertSame( + strlen($packed) + strlen('value'), + MutationBudget::entrySize([$subspace->pack(['user', 42]), 'value']), + ); + } + + // -- constants --------------------------------------------------------------- + + #[Test] + public function splitTargetIsSafelyBelowTheHardLimit(): void + { + // A split group may overshoot its target by at most one entry + // (max key + max value), and must still stay under the hard limit. + $worstCase = MutationBudget::SPLIT_TARGET_BYTES + + KeyValueLimits::MAX_KEY_SIZE + + KeyValueLimits::MAX_VALUE_SIZE; + + self::assertLessThan(MutationBudget::TRANSACTION_MUTATION_LIMIT, $worstCase); + } +}