Skip to content

fix(test-utils): share one network namespace across cluster nodes #3325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
GiHoon1123 wants to merge 3 commits into from
+140 −95
Closed

fix(test-utils): share one network namespace across cluster nodes #3325

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c0c27f7
fix(test-utils): share one network namespace across cluster nodes
GiHoon1123 Jul 9, 2026
0b79b5b
fix(test-utils): don't publish cluster bus ports to the host
GiHoon1123 Jul 9, 2026
ad2b5bc
fix(test-utils): keep the announced cluster bus port within valid range
GiHoon1123 Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 140 additions & 95 deletions packages/test-utils/lib/dockers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import { createConnection } from 'node:net';
import { once } from 'node:events';
import { createClient } from '@redis/client/index';
import { setTimeout } from 'node:timers/promises';
// import { ClusterSlotsReply } from '@redis/client/dist/lib/commands/CLUSTER_SLOTS';
import { execFile as execFileCallback } from 'node:child_process';
import { promisify } from 'node:util';
import * as fs from 'node:fs';
Expand Down Expand Up @@ -444,90 +443,87 @@ export type RedisClusterDockersConfig = RedisServerDockerOptions & {
numberOfReplicas?: number;
}

async function spawnRedisClusterNodeDockers(
dockersConfig: RedisClusterDockersConfig,
serverArguments: Array<string>,
fromSlot: number,
toSlot: number,
clientConfig?: Partial<RedisClusterClientOptions>
) {
const range: Array<number> = [];
for (let i = fromSlot; i < toSlot; i++) {
range.push(i);
}

const master = await spawnRedisClusterNodeDocker(
dockersConfig,
serverArguments,
clientConfig
);
const CLUSTER_BUS_PORT_OFFSET = 10000;
// Cluster nodes announce `port + CLUSTER_BUS_PORT_OFFSET` as their bus port
// (see spawnClusterNetworkNode below), so client ports reserved for a
// cluster must leave enough headroom for that to stay a valid TCP port.
const MAX_CLUSTER_CLIENT_PORT = 65535 - CLUSTER_BUS_PORT_OFFSET;

await master.client.clusterAddSlots(range);
async function reservePorts(count: number, maxPort = 65535): Promise<number[]> {
const ports: number[] = [];
for (let i = 0; i < count; i++) {
let port: number;
do {
port = (await portIterator.next()).value;
} while (port > maxPort);
ports.push(port);
}
return ports;
}

if (!dockersConfig.numberOfReplicas) return [master];
// Spawns one redis-server node that's part of a cluster where *all* nodes
// share a single Docker network namespace, instead of `--network host`.
//
// Why: on macOS/Windows, `--network host` only reaches the Docker Desktop VM,
// not the real host, so the host test process can't connect to the nodes at
// all (see #2358). Publishing ports (`-p`) the usual way fixes host access,
// but Redis Cluster nodes can only *announce* one address for themselves --
// an address that's valid for sibling containers (their bridge-network IP)
// is *not* valid for the host, and vice-versa.
//
// Fix: put every node in the cluster in the *same* network namespace via
// `--network container:<id>`. The first node spawned ("owner") gets a normal
// network and publishes every port the whole cluster will ever use; every
// other node joins the owner's namespace instead of getting its own. From
// then on, `127.0.0.1:<port>` means the same thing to the host *and* to every
// sibling node, so `--cluster-announce-ip 127.0.0.1` is simultaneously valid
// for both audiences.
async function spawnClusterNetworkNode(
dockersConfig: RedisServerDockerOptions,
serverArguments: Array<string>,
port: number,
ownerPorts: number[] | undefined,
networkOwnerId: string | undefined
): Promise<RedisServerDocker> {
const portStr = port.toString();
const busPort = port + CLUSTER_BUS_PORT_OFFSET;

const replicasPromises: Array<ReturnType<typeof spawnRedisClusterNodeDocker>> = [];
for (let i = 0; i < (dockersConfig.numberOfReplicas ?? 0); i++) {
replicasPromises.push(
spawnRedisClusterNodeDocker(dockersConfig, [
...serverArguments,
'--cluster-enabled',
'yes',
'--cluster-node-timeout',
'5000'
], clientConfig).then(async replica => {
const dockerArgs = ['run', '--init', '-e', `PORT=${portStr}`];

const requirePassIndex = serverArguments.findIndex((x) => x === '--requirepass');
if (requirePassIndex !== -1) {
const password = serverArguments[requirePassIndex + 1];
await replica.client.configSet({ 'masterauth': password })
}
await replica.client.clusterMeet('127.0.0.1', master.docker.port);
if (networkOwnerId) {
dockerArgs.push('--network', `container:${networkOwnerId}`);
} else {
// Only the client port needs to be reachable from the host -- the
// cluster bus port is only ever dialed by sibling nodes over loopback
// inside this shared namespace, so it doesn't need (and shouldn't get)
// a host mapping. Publishing it anyway is both pointless and unsafe:
// `port + CLUSTER_BUS_PORT_OFFSET` can exceed 65535 for high ports,
// which makes Docker reject the whole `docker run`.
for (const p of ownerPorts!) {
dockerArgs.push('-p', `${p}:${p}`);
}
}

while ((await replica.client.clusterSlots()).length === 0) {
await setTimeout(25);
}
dockerArgs.push('-d', `${dockersConfig.image}:${dockersConfig.version}`);
dockerArgs.push(
...serverArguments,
'--cluster-announce-ip', '127.0.0.1',
'--cluster-announce-port', portStr,
'--cluster-announce-bus-port', busPort.toString()
);

await replica.client.clusterReplicate(
await master.client.clusterMyId()
);
console.log(`[Docker] Spawning cluster node - Port: ${port}, network: ${networkOwnerId ? `container:${networkOwnerId}` : '(owner)'}`);

return replica;
})
);
const { stdout, stderr } = await execAsync('docker', dockerArgs);
if (!stdout) {
throw new Error(`docker run error - ${stderr}`);
}

return [
master,
...await Promise.all(replicasPromises)
];
}

async function spawnRedisClusterNodeDocker(
dockersConfig: RedisServerDockerOptions,
serverArguments: Array<string>,
clientConfig?: Partial<RedisClusterClientOptions>
) {
const docker = await spawnRedisServerDocker(dockersConfig, [
...serverArguments,
'--cluster-enabled',
'yes',
'--cluster-node-timeout',
'5000'
]),
client = createClient({
socket: {
port: docker.port
},
...clientConfig
});

await client.connect();
while (await isPortAvailable(port)) {
await setTimeout(50);
}

return {
docker,
client
};
return { port, dockerId: stdout.trim() };
}

const SLOTS = 16384;
Expand All @@ -538,50 +534,99 @@ async function spawnRedisClusterDockers(
clientConfig?: Partial<RedisClusterClientOptions>
): Promise<Array<RedisServerDocker>> {
const numberOfMasters = dockersConfig.numberOfMasters ?? 2,
slotsPerNode = Math.floor(SLOTS / numberOfMasters),
spawnPromises: Array<ReturnType<typeof spawnRedisClusterNodeDockers>> = [];
numberOfReplicas = dockersConfig.numberOfReplicas ?? 0,
totalNodeCount = numberOfMasters * (1 + numberOfReplicas),
slotsPerNode = Math.floor(SLOTS / numberOfMasters);

// All ports must be known up-front: the network "owner" node has to
// publish every port the cluster will use in its single `docker run`.
const ports = await reservePorts(totalNodeCount, MAX_CLUSTER_CLIENT_PORT);
const nodeArguments = [...serverArguments, '--cluster-enabled', 'yes', '--cluster-node-timeout', '5000'];

const ownerDocker = await spawnClusterNetworkNode(dockersConfig, nodeArguments, ports[0], ports, undefined);
const restDockers = await Promise.all(
ports.slice(1).map(port =>
spawnClusterNetworkNode(dockersConfig, nodeArguments, port, undefined, ownerDocker.dockerId)
)
);
const dockers = [ownerDocker, ...restDockers];

const nodes = await Promise.all(
dockers.map(async docker => {
const client = createClient({ socket: { port: docker.port }, ...clientConfig });
await client.connect();
return { docker, client };
})
);

// First `numberOfMasters` nodes are masters (one per shard); the rest are
// replicas, handed out round-robin across the masters.
const masters = nodes.slice(0, numberOfMasters);
const replicasByMaster: (typeof nodes)[] = masters.map(() => []);
for (let i = numberOfMasters; i < nodes.length; i++) {
replicasByMaster[(i - numberOfMasters) % numberOfMasters].push(nodes[i]);
}

for (let i = 0; i < numberOfMasters; i++) {
const fromSlot = i * slotsPerNode,
toSlot = i === numberOfMasters - 1 ? SLOTS : fromSlot + slotsPerNode;
spawnPromises.push(
spawnRedisClusterNodeDockers(
dockersConfig,
serverArguments,
fromSlot,
toSlot,
clientConfig
)
);
toSlot = i === numberOfMasters - 1 ? SLOTS : fromSlot + slotsPerNode,
range: Array<number> = [];
for (let s = fromSlot; s < toSlot; s++) range.push(s);
await masters[i].client.clusterAddSlots(range);
}

const nodes = (await Promise.all(spawnPromises)).flat(),
meetPromises: Array<Promise<unknown>> = [];
const meetPromises: Array<Promise<unknown>> = [];
for (let i = 1; i < nodes.length; i++) {
meetPromises.push(
nodes[i].client.clusterMeet('127.0.0.1', nodes[0].docker.port)
);
}

await Promise.all(meetPromises);

// Wait for gossip to cover all slots before replicating -- this only needs
// the masters to be visible, not the (not-yet-configured) replicas, so it
// must happen *before* clusterReplicate below, not after.
await Promise.all(
nodes.map(async ({ client }) => {
while (
totalNodes(await client.clusterSlots()) !== nodes.length ||
!(await client.sendCommand<string>(['CLUSTER', 'INFO'])).startsWith('cluster_state:ok') // TODO
) {
await setTimeout(50);
}
})
);

for (let i = 0; i < numberOfMasters; i++) {
for (const replica of replicasByMaster[i]) {
const requirePassIndex = serverArguments.findIndex((x) => x === '--requirepass');
if (requirePassIndex !== -1) {
const password = serverArguments[requirePassIndex + 1];
await replica.client.configSet({ 'masterauth': password });
}
await replica.client.clusterReplicate(await masters[i].client.clusterMyId());
}
}

client.destroy();
// Now that replicas are attached, wait for every node to see the full
// node count (masters + replicas) before handing the cluster back.
await Promise.all(
nodes.map(async ({ client }) => {
while (totalNodes(await client.clusterSlots()) !== nodes.length) {
await setTimeout(50);
}
})
);

return nodes.map(({ docker }) => docker);
for (const { client } of nodes) {
client.destroy();
}

return dockers;
}

// TODO: type ClusterSlotsReply
function totalNodes(slots: any) {
type ClusterSlotsReply = Awaited<ReturnType<ReturnType<typeof createClient>['clusterSlots']>>;

function totalNodes(slots: ClusterSlotsReply) {
let total = slots.length;
for (const slot of slots) {
total += slot.replicas.length;
Expand Down