diff --git a/Cargo.lock b/Cargo.lock index c543423ce..e70180b38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1318,7 +1318,7 @@ dependencies = [ [[package]] name = "flecs_ecs" version = "0.2.2" -source = "git+https://github.com/andrewgazelka/Flecs-Rust?rev=252944dedbc80741b7cca30dea67c5be95638950#252944dedbc80741b7cca30dea67c5be95638950" +source = "git+https://github.com/andrewgazelka/Flecs-Rust?rev=f09dc5308d00c6a88c82b1195334b6ed2b2d2868#f09dc5308d00c6a88c82b1195334b6ed2b2d2868" dependencies = [ "bitflags 2.13.1", "compact_str", @@ -1332,7 +1332,7 @@ dependencies = [ [[package]] name = "flecs_ecs_derive" version = "0.2.0" -source = "git+https://github.com/andrewgazelka/Flecs-Rust?rev=252944dedbc80741b7cca30dea67c5be95638950#252944dedbc80741b7cca30dea67c5be95638950" +source = "git+https://github.com/andrewgazelka/Flecs-Rust?rev=f09dc5308d00c6a88c82b1195334b6ed2b2d2868#f09dc5308d00c6a88c82b1195334b6ed2b2d2868" dependencies = [ "proc-macro2", "quote", @@ -1342,7 +1342,7 @@ dependencies = [ [[package]] name = "flecs_ecs_sys" version = "0.2.1" -source = "git+https://github.com/andrewgazelka/Flecs-Rust?rev=252944dedbc80741b7cca30dea67c5be95638950#252944dedbc80741b7cca30dea67c5be95638950" +source = "git+https://github.com/andrewgazelka/Flecs-Rust?rev=f09dc5308d00c6a88c82b1195334b6ed2b2d2868#f09dc5308d00c6a88c82b1195334b6ed2b2d2868" dependencies = [ "bindgen", "cc", @@ -2022,6 +2022,23 @@ dependencies = [ "hyperion-hot-reload", ] +[[package]] +name = "hyperion-hot-reload-index-probe" +version = "0.1.0" +dependencies = [ + "flecs_ecs", + "hyperion", + "libloading 0.9.0", +] + +[[package]] +name = "hyperion-hot-reload-index-probe-module" +version = "0.1.0" +dependencies = [ + "flecs_ecs", + "hyperion", +] + [[package]] name = "hyperion-inventory" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 31ae507f5..7d10c0878 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,8 @@ members = [ 'crates/hyperion-gui', 'crates/hyperion-hot-reload', 'crates/hyperion-hot-reload/demo/host', + 'crates/hyperion-hot-reload/demo/index-probe-host', + 'crates/hyperion-hot-reload/demo/index-probe-module', 'crates/hyperion-hot-reload/demo/module', 'crates/hyperion-inventory', 'crates/hyperion-item', @@ -180,9 +182,15 @@ version = "1.1.9" # stage to `flecs_components_get`, which asserts, so every system with a sparse # term aborts under `set_threads > 1` — which is every system here. Repin on # upstream once that lands. +# +# Also carrying the shared-dylib change: `crate-type = ["dylib", "rlib"]` plus a +# build script that re-exports flecs's C symbols. Hot reloading needs exactly one +# copy of this crate in the process, because it owns the process-global pool that +# hands out each component type's index into a world's component array. See +# `docs/hot-reload.md`. features = ['flecs_manual_registration'] git = 'https://github.com/andrewgazelka/Flecs-Rust' -rev = '252944dedbc80741b7cca30dea67c5be95638950' +rev = 'f09dc5308d00c6a88c82b1195334b6ed2b2d2868' [workspace.dependencies.geometry] path = 'crates/geometry' diff --git a/crates/hyperion-hot-reload/build.rs b/crates/hyperion-hot-reload/build.rs index 633ad5967..5f93a3de7 100644 --- a/crates/hyperion-hot-reload/build.rs +++ b/crates/hyperion-hot-reload/build.rs @@ -35,16 +35,64 @@ fn record_rustc() { println!("cargo::rustc-env=HYPERION_HOT_RELOAD_RUSTC={fingerprint}"); } +/// The flecs C symbol patterns both platforms have to re-export, spelled without the +/// leading underscore Mach-O adds. +/// +/// `flecs_ecs`'s own `build.rs` carries the same list, and that is not a duplicate to +/// consolidate: every dylib that ends up *containing* flecs's C has to export it, and +/// which dylib that is depends on how the consumer links. Under the shared-dylib recipe +/// flecs lives in `libflecs_ecs.so` and this list matches nothing here, harmlessly; built +/// without `-C prefer-dynamic`, this crate absorbs flecs itself and this list is the only +/// thing making it reachable. +const FLECS_EXPORTS: [&str; 4] = ["ecs_*", "flecs_*", "Ecs*", "FLECS_*"]; + fn export_flecs_symbols() { println!("cargo::rerun-if-changed=build.rs"); let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); if matches!(target_os.as_str(), "macos" | "ios") { println!("cargo::rustc-link-arg=-Wl,-all_load"); // ld64 unions -exported_symbol with the export list rustc generates. - for pattern in ["_ecs_*", "_flecs_*", "_Ecs*", "_FLECS_*"] { - println!("cargo::rustc-link-arg=-Wl,-exported_symbol,{pattern}"); + for pattern in FLECS_EXPORTS { + println!("cargo::rustc-link-arg=-Wl,-exported_symbol,_{pattern}"); } } else { - println!("cargo::rustc-link-arg=-Wl,--export-dynamic"); + export_flecs_symbols_elf(); + } +} + +/// ELF needs a version script, because `--export-dynamic` cannot undo what rustc does. +/// +/// rustc links a `dylib` with its own anonymous version script ending in `local: *`, which +/// demotes every symbol it did not generate. flecs's C symbols land in the object with +/// `DEFAULT` visibility and `LOCAL` binding, so they are present and unreachable, and +/// `--export-dynamic` and `--export-dynamic-symbol` are both powerless against a +/// version-script demotion. Measured on x86_64-linux: 9001 exported symbols, zero of them +/// `ecs_*`, `ecs_init` reading `FUNC LOCAL DEFAULT`. +/// +/// A game module that cannot resolve `ecs_*` here links its own copy of `libflecs.a` +/// instead, which is two `ecs_os_api` globals in one process -- the failure `AbiToken` +/// exists to catch, arriving on a platform where the check itself could not run. +/// +/// ld merges multiple version scripts and an explicit pattern beats a `*` wildcard, so a +/// second script naming these globs promotes exactly them and leaves rustc's own exports +/// alone. Same measurement after: 10717 exported, 666 of them `ecs_*`, `ecs_init` GLOBAL. +fn export_flecs_symbols_elf() { + let out_dir = std::env::var("OUT_DIR").expect("cargo always sets OUT_DIR"); + let script = std::path::Path::new(&out_dir).join("flecs-exports.map"); + + let mut text = String::from("{\n global:\n"); + for pattern in FLECS_EXPORTS { + text.push_str(" "); + text.push_str(pattern); + text.push_str(";\n"); } + // No `local:` clause. This script adds to rustc's export list rather than replacing + // it; a `local: *` here would hide every Rust symbol the host resolves through. + text.push_str("};\n"); + + std::fs::write(&script, text).expect("failed to write the flecs version script"); + println!( + "cargo::rustc-link-arg=-Wl,--version-script={}", + script.display() + ); } diff --git a/crates/hyperion-hot-reload/demo/index-probe-host/Cargo.toml b/crates/hyperion-hot-reload/demo/index-probe-host/Cargo.toml new file mode 100644 index 000000000..3b80177dd --- /dev/null +++ b/crates/hyperion-hot-reload/demo/index-probe-host/Cargo.toml @@ -0,0 +1,19 @@ +[package] +edition.workspace = true +license.workspace = true +name = "hyperion-hot-reload-index-probe" +publish = false +repository.workspace = true +version.workspace = true + +[[bin]] +name = "hot-reload-index-probe" +path = "src/main.rs" + +[dependencies] +flecs_ecs.workspace = true +hyperion.workspace = true +libloading = "0.9.0" + +[lints] +workspace = true diff --git a/crates/hyperion-hot-reload/demo/index-probe-host/src/main.rs b/crates/hyperion-hot-reload/demo/index-probe-host/src/main.rs new file mode 100644 index 000000000..785a178c3 --- /dev/null +++ b/crates/hyperion-hot-reload/demo/index-probe-host/src/main.rs @@ -0,0 +1,71 @@ +//! Compares component-index allocation across the host/module dylib boundary. +//! See the module crate for why this is behavioural rather than an address comparison. +#![allow( + clippy::print_stdout, + reason = "this binary's whole purpose is reporting what it measured" +)] + +use flecs_ecs::core::ComponentId; + +#[derive(flecs_ecs::macros::Component)] +struct HostMarkerA; +#[derive(flecs_ecs::macros::Component)] +struct HostMarkerB; +#[derive(flecs_ecs::macros::Component)] +struct HostMarkerC; + +fn main() { + let path = std::env::args() + .nth(1) + .expect("usage: hot-reload-index-probe "); + + // Take several indices before the module is even loaded, so a shared pool has visibly + // advanced by the time the module asks for one. + let host_indices = [ + ::index(), + ::index(), + ::index(), + ::index(), + ]; + let host_max = host_indices.iter().copied().max().expect("non-empty"); + + let lib = unsafe { libloading::Library::new(&path) }.expect("failed to dlopen the module"); + let module_index = unsafe { + let f: libloading::Symbol<'_, unsafe extern "C" fn() -> u32> = lib + .get(b"probe_module_index") + .expect("no module index symbol"); + f() + }; + let module_position_index = unsafe { + let f: libloading::Symbol<'_, unsafe extern "C" fn() -> u32> = lib + .get(b"probe_position_index") + .expect("no position index symbol"); + f() + }; + // Leaked deliberately: `dlclose` would unmap text the process still holds pointers + // into, which is the segfault-at-exit documented in docs/hot-reload.md. + core::mem::forget(lib); + + println!("host indices: {host_indices:?} (max {host_max})"); + println!("module's own type index: {module_index}"); + println!( + "hyperion::simulation::Position index: host {}, module {module_position_index}", + host_indices[3] + ); + + let shared_pool = module_index > host_max; + let shared_hyperion_index = host_indices[3] == module_position_index; + println!("SHARED_POOL={shared_pool}"); + println!("SHARED_HYPERION_INDEX={shared_hyperion_index}"); + + assert!( + shared_pool, + "host and module allocate component indices from separate pools: the module got \ + {module_index} after the host had already taken up to {host_max}.\nThis is the expected \ + result on a default build, and the probe is the reason to know it. Passing needs the \ + dylib recipe in docs/hot-reload.md: `hyperion` built as a dylib and everything compiled \ + with `-C prefer-dynamic -C link-arg=-Wl,--undefined-version -C \ + link-arg=-Wl,--allow-shlib-undefined`." + ); + println!("PROBE_OK"); +} diff --git a/crates/hyperion-hot-reload/demo/index-probe-module/Cargo.toml b/crates/hyperion-hot-reload/demo/index-probe-module/Cargo.toml new file mode 100644 index 000000000..140445d72 --- /dev/null +++ b/crates/hyperion-hot-reload/demo/index-probe-module/Cargo.toml @@ -0,0 +1,17 @@ +[package] +edition.workspace = true +license.workspace = true +name = "hyperion-hot-reload-index-probe-module" +publish = false +repository.workspace = true +version.workspace = true + +[lib] +crate-type = ["dylib"] + +[dependencies] +flecs_ecs.workspace = true +hyperion.workspace = true + +[lints] +workspace = true diff --git a/crates/hyperion-hot-reload/demo/index-probe-module/src/lib.rs b/crates/hyperion-hot-reload/demo/index-probe-module/src/lib.rs new file mode 100644 index 000000000..fab263a4a --- /dev/null +++ b/crates/hyperion-hot-reload/demo/index-probe-module/src/lib.rs @@ -0,0 +1,50 @@ +//! Answers one question: do a host binary and a module dylib draw component indices from +//! one shared pool? +//! +//! `flecs_ecs`'s derive emits, per component type, a `static INDEX` initialised from a +//! process-global `INDEX_POOL`, and that index is a slot in the world's component array. +//! Two copies of `flecs_ecs` in one process means two pools, so the module writes into a +//! slot the host never filled. Everything the hot-reload gate does rests on this being one +//! pool, and nothing else it checks would notice if it were not. +//! +//! The test is behavioural rather than an address comparison, deliberately. Comparing +//! `ecs_init as usize` across the boundary reports a difference even when the copy is +//! shared, because an executable taking the address of a dynamically-linked function gets +//! its own PLT stub rather than the implementation. Measured exactly that trap: separate +//! copies and shared copies both printed mismatched addresses. Allocation order cannot be +//! faked -- if the pool is shared, an index taken here is strictly greater than every +//! index the host took first. +//! +//! The equality of any single index is not evidence either, and looked like evidence once. +//! Two separate pools both start at 1, so a type that happens to be the first registered on +//! each side reads `1` and `1`. That is what this probe reported when the module linked its +//! own static copy of everything, which is exactly the case it exists to detect. +//! +//! What makes the pool shared is `flecs_ecs` being built as a dylib, so every consumer +//! resolves one `libflecs_ecs.so`. It is not this crate's dependency list: dropping the +//! `hyperion-hot-reload` dependency entirely leaves the probe passing. + +use flecs_ecs::core::ComponentId; + +/// Declared here so its index can only ever have been allocated by this dylib. +#[derive(flecs_ecs::macros::Component)] +pub struct ModuleOnlyMarker; + +/// A component type `hyperion` owns, to check the shared-pool result holds for a type +/// neither side declares locally. +/// +/// # Safety +/// Called by the probe host through `dlsym`. Returns a plain integer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn probe_position_index() -> u32 { + ::index() +} + +/// An index allocated in this dylib, after the host has already taken several. +/// +/// # Safety +/// Called by the probe host through `dlsym`. Returns a plain integer. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn probe_module_index() -> u32 { + ::index() +} diff --git a/docs/hot-reload.md b/docs/hot-reload.md index 299d438bd..faeaedec6 100644 --- a/docs/hot-reload.md +++ b/docs/hot-reload.md @@ -418,5 +418,276 @@ Stated plainly, because these are the parts a reader cannot see for themselves. duplicate entries, and this branch's lock differs from it only by the three new crates. The two hot-reload checks build and pass on their own: `nix build .#checks..hot-reload-demo .#checks..hot-reload-registry-guard`. -- **Only tested on aarch64-darwin.** `build.rs` has a Linux branch using - `--export-dynamic` instead of ld64's `-exported_symbol`, and it has never been run. +- **The deployment half is designed, not shipped.** Nothing here is wired to `ix apply`, + to a systemd unit, or to hyperion's own modules. See "Deploying a reload" below for the + shape and what is missing. + +## Linux, and the one copy of flecs everything depends on + +Verified on x86_64-linux (dev-compute-6, rustc 1.99.0-nightly `dc3f85158`). It did not +work there before, in two separate ways, and both failures are quiet enough to be worth +naming. + +### Exports: `--export-dynamic` cannot undo what rustc does to a dylib + +rustc links a `dylib` with its own anonymous version script ending in `local: *`, which +demotes every symbol it did not generate. flecs's C symbols arrive with `DEFAULT` +visibility and `LOCAL` binding — physically present, dynamically unreachable: + +``` +$ nm -D --defined-only libhyperion_hot_reload.so | wc -l +9001 +$ nm -D --defined-only libhyperion_hot_reload.so | grep -c " ecs_" +0 +$ readelf -sW libhyperion_hot_reload.so | grep -w ecs_init + 5905: ... FUNC LOCAL DEFAULT 14 ecs_init +``` + +Neither `-Wl,--export-dynamic` nor `-Wl,--export-dynamic-symbol=ecs_*` helps; a +version-script demotion is not something either flag can reverse. Both were tried and both +left the count at 0. The fix is a *second* version script naming the flecs globs with no +`local:` clause — ld merges version scripts and an explicit pattern beats a `*` wildcard, +so it promotes exactly those and leaves rustc's own exports alone. After: 10717 exported, +666 of them `ecs_*`, `ecs_init` `GLOBAL`. + +Note that `-all_load` was never the load-bearing half on macOS either. `flecs_ecs_sys` +compiles `src/flecs_rust.c`, which `#include`s `flecs.c`, so `libflecs.a` has exactly one +member and any reference drags the whole thing in on both platforms. The platforms differ +only in export visibility. + +**Where the script goes matters and the wrong placement is silent.** A build script's +`rustc-link-arg` applies to its own crate's artifacts. Putting it in `flecs_ecs_sys` — the +crate flecs's C actually lives in — does nothing, because that crate is an rlib absorbed +into a dylib rather than linked itself. Measured: 0 exported `ecs_*`, no warning. It +belongs in whichever crate *produces the dylib*. + +### One pool, or the world is indexed two different ways + +`flecs_ecs`'s derive emits, per component type, a `static INDEX` initialised from a +process-global `INDEX_POOL`, and that index is a slot in the world's component array. The +`flecs_manual_registration` note earlier in this document is about the *id* being +per-world; the *index* is not. Two copies of `flecs_ecs` in one process is two pools, and +a module then writes into a slot the host never filled. + +Nothing in the reload path detects this. `AbiToken` passes, no error is raised, and both +sides are internally consistent — they simply disagree about which slot is which. + +`demo/index-probe-{host,module}` measures it. Two things about how, because the obvious +approaches both give the wrong answer: + +- **Do not compare `ecs_init as usize` across the boundary.** An executable taking the + address of a dynamically-linked function gets its own PLT stub, so the addresses differ + whether or not the copy is shared. Measured both cases; both printed a mismatch. +- **Do not compare one type's index for equality.** Two independent pools each start at 1, + so the first type registered on each side reads `1` and `1` and looks shared when nothing + is. This is what the probe reported before the module referenced the runtime crate at all. + +What works is allocation order, which cannot coincide: with one pool, an index taken in the +module is strictly greater than every index the host took first. + +With `hyperion` as a plain rlib: + +``` +host indices: [1, 2, 3, 4] (max 4) +module's own type index: 1 +hyperion::simulation::Position index: host 4, module 2 +SHARED_POOL=false +``` + +The *host* is what creates the second copy — it pulls `flecs_ecs` in through hyperion's +rlib while the module resolves it from the runtime dylib. A module that touches no hyperion +component does not avoid this; the host's own linkage is enough. + +With `flecs_ecs` and `hyperion` both dylibs: + +``` +host indices: [1, 2, 3, 4] (max 4) +module's own type index: 5 +hyperion::simulation::Position index: host 4, module 4 +SHARED_POOL=true +SHARED_HYPERION_INDEX=true +PROBE_OK +``` + +### A behaviour-only module does not avoid this + +There is an appealing argument that it should. `CLAUDE.md` splits every flecs module into a +registration module that only declares components and a behaviour module that only installs +systems and observers, and notes that this lets a consumer import the types without the +systems. Put only behaviour modules in the reloadable library, keep every registration +module in the host, and the manual-registration problem does look like it disappears: a +library that registers nothing cannot collide with anything. + +That much is true, and it is the right split for a different reason given below. **It does +not make the shared pool optional.** Registering a component and *looking one up* are +different operations and only the first is avoided. A system's query still resolves `T` to +an id through `T::index()`, so a behaviour module reading `Position` needs the same index +the host filled. + +Measured, not argued. `demo/index-probe-module` registers nothing at all — it declares one +marker type and calls `index()` — and in the unshared configuration it read +`hyperion::simulation::Position` as index **2** where the host had it at **4**. A pure +behaviour module querying `Position` would have read a slot the host never wrote. + +So the registration/behaviour split is worth keeping, but for the hazard it actually +addresses: **component layout**. A system compiled against one struct layout reading a world +that holds another is silent memory corruption. Keeping component definitions in the host +and having the library depend on them rather than declare them means there is exactly one +definition of each layout in the process. The gate on top of that turns a layout change into +a refusal rather than a corruption. + +The honest boundary that falls out, and which belongs in front of anyone using this: +**changing what a system does is a reload; adding or changing a component type is a host +rebuild and a restart.** + +### The build recipe + +1. `flecs_ecs` needs `crate-type = ["dylib", "rlib"]` and a `build.rs` emitting the version + script above. Without the dylib you get + `error: cannot satisfy dependencies so 'flecs_ecs' only shows up once`, because two + dylibs each bundle their own copy. +2. `hyperion` needs `crate-type = ["dylib", "rlib"]`. + + Both crates now emit two artifacts on every build rather than one -- `flecs_ecs`'s + dylib is about 37 MB next to a 45 MB rlib. Whether that costs meaningful build time is + **not established**: a clean `cargo build -p flecs_ecs` measured 8875 ms with both and + 8817 ms with the rlib alone, which is within noise, but a stale dylib in the target + directory means the second configuration may not have taken effect. Treat the build-time + cost as unmeasured rather than as shown to be zero, and measure it properly if CI wall + time matters. +3. Host and every module build with + `-C prefer-dynamic -C link-arg=-Wl,--undefined-version -C link-arg=-Wl,--allow-shlib-undefined`, + plus rpaths to the rust sysroot and to wherever the dylibs land. + +What makes the pool shared is step 1 and nothing else. It is tempting to think a module has +to *reference* `hyperion-hot-reload` to end up on the shared runtime — an earlier version of +the probe carried a call to `AbiToken::current()` with a comment claiming exactly that. +Removing the dependency entirely leaves the probe passing. The dependency being a dylib is +what shares it; a consumer's import list has nothing to do with it. + +`--allow-shlib-undefined` is not a shrug. `simulation/metadata/mod.rs` hand-writes +`impl PartialOrd for $name where $type: PartialOrd`, and for 7 metadata types that bound is +unsatisfiable because glam's `Quat` and `Vec3` have no `PartialOrd`. rustc never codegens +those `partial_cmp` bodies but still lists them in the dylib's export list. They cannot be +called — calling one fails to compile on the same unsatisfiable bound — so allowing them +undefined is sound. Removing the blanket impl from that macro would remove the need for the +flag, and is the better fix. + +**Steps 1 and 2 are not landed.** They were verified through a local `[patch]` against a +copy of the fork checkout. Landing them means a commit in `andrewgazelka/Flecs-Rust` and a +repin here. + +## Deploying a reload + +The mechanism a running server needs is not a file watcher. It is systemd's, and NixOS +already exposes it. + +`nixos/doc/manual/development/unit-handling.section.md` in nixpkgs: *"If they are different +but only `X-Reload-Triggers` in the `[Unit]` section is changed, **reload** the unit."* So a +game-logic-only change can reach a running server as a `systemctl reload` rather than a +restart, which means the process never exits, the proxy's backend socket never closes, and +no player is disturbed. + +Three pieces make that hold: + +- `reloadTriggers = [ gameModuleDylib ]`, so the dylib's store path lands in + `X-Reload-Triggers` **and nowhere else**. If it also appeared in `ExecStart` or + `Environment` the `[Service]` section would differ and the whole scheme degrades to a + restart. +- The process reaches the dylib through a stable path — `environment.etc` — since `/etc` is + rebuilt during activation, before units are acted on, and changing a symlink there is not + a unit change at all. +- `ExecReload` is a fixed string: a client that asks the running process to reload and + **exits non-zero when the gate refuses**, printing the refusal and its `migration!` stub. + A refused reload then surfaces as a failed activation with the reason in the deploy + output, rather than as a silent no-op, and the world keeps running on the old build. + +Not built. `app.run()` in an event's `init_game` is flecs's own main loop and offers no +per-tick Rust hook; it would become an explicit `while world.progress()` so reloads land +between ticks, which is also what the "reloads must happen between ticks" gap above needs. + +### What a reload costs + +Measured, debug profile, three runs each. The Linux figures are dev-compute-6 (32 cores); +the smash figure is aarch64-darwin. + +| | | +| --- | --- | +| rules-only rebuild and link (`smash`, 9756 lines of rules code, one line edited) | 1.76 / 1.85 / 2.07 s | +| minimal module rebuild and link (165 KB dylib) | 282 / 285 / 283 ms | +| process start **and** `dlopen` — an upper bound on the reload | 31 ms | +| touching the engine instead: host and engine rebuild | 4.42 s | +| unit restart | zero, by construction | +| world rebuild, chunk resend, re-join | zero, by construction | + +The last two rows are the point. An engine change costs 2.4x the compile *and* loses the +process, the world and every connected player. A rules change costs neither. + +Three things these numbers are not: + +- **Not the release profile.** A deployment builds release, which compiles slower. Treat + these as the shape of the cost, not the deployed figure. +- **Not a rules dylib.** The rules are not a separate crate yet, so the smash row is + `-p smash` relinking the whole binary. A rules dylib links strictly less, so this + over-estimates rather than under-estimates. +- **Not smash's reload.** The 31 ms is a 165 KB probe module and includes process startup, + so the in-process reload is strictly less — but a larger dylib takes longer to `dlopen`, + and the schema diff scales with component count. The demo separately reports + `instances rewritten: 0` for a code-only change, which is the case that does no archetype + moves at all. + +For the deployment as a whole, the game server is not the slow part. Most of an `ix apply` +is working out what to deploy rather than deploying it, and that cost is unrelated to +anything here. + +## Handing this off: what is left, in order + +The mechanism is proven and the deployment is not built. Four steps remain. The third is +the risky one; the rest are known work. + +**1. Make `hyperion` a dylib and settle the build flags.** `crate-type = ["dylib", "rlib"]` +plus `-C prefer-dynamic -C link-arg=-Wl,--undefined-version +-C link-arg=-Wl,--allow-shlib-undefined` everywhere. Small edit, wide blast radius: it +changes how every consumer links, and a plain `cargo test` without those flags will not +link the result. Confirm by running `demo/index-probe-host`, which should print `PROBE_OK`. + +**2. Split `SmashModule` out of `events/smash` into its own crate, built as a dylib with +`export_module!`.** The rules already avoid the host seam by design, but they reach into +`crate::server`, `crate::flecs_ext` and about fifteen `hyperion::` items, so this is a real +refactor rather than a file move. Registration modules stay in the host per the section +above. + +**3. Package it. This is the risky step.** The game server binary and the module dylib have +to be separate store paths, both built with the flags from step 1, with rpaths that resolve +in the nix store rather than in `target/debug`. Nothing here is verified — every +measurement in this document was taken from a cargo build, not a nix one. Expect the +surprises to be here. + +**4. Wire the NixOS module and the fleet spec.** Designed in "Deploying a reload" above: +`reloadTriggers`, the stable `/etc` path, and an `ExecReload` client that exits non-zero on +a refusal. Small, and the design is settled. + +### Adopting it costs no scheduled restart + +Step 1 changes the host binary, so the running game server has to restart once to pick it +up — but that restart never has to be scheduled *for this*. The fleet already restarts for +version bumps, and the proxy and the game are built from the same repository, so those +move every node anyway. The split host can sit in the tree and take effect on the next +apply that was happening regardless. + +Design for that: none of this should want its own window. The claim is then not "one +restart, then none" but that no restart was ever scheduled for it, including the first. + +### Reproducing the measurements + +Everything in this document was measured on dev-compute-6 (x86_64-linux, 32 cores, +rustc 1.99.0-nightly `dc3f85158`) and on aarch64-darwin, through `nix develop` and cargo. +The build tree under `/tmp/hotreload-elf` on that host **was deleted** when the node was +released, so reproducing means a fresh clone and a warm toolchain fetch — roughly twenty +seconds for the devShell once the store is warm, and a couple of minutes for the first +`cargo build -p hyperion-hot-reload`. + +Note that `cargo build -p smash` does **not** work in the devShell on Linux, for reasons +unrelated to any of this: jemalloc 5.3.1's configure cannot survive GCC 15 +(`cannot determine return type of strerror_r`). The nix package path is unaffected. +ENG-11279. Until it is fixed, iterate on darwin and use nix for Linux artifacts. diff --git a/flake.nix b/flake.nix index 24b5ba18a..df62148d6 100644 --- a/flake.nix +++ b/flake.nix @@ -1016,8 +1016,8 @@ outputHashes = { "git+https://github.com/nvzqz/divan#55ec68e31526c28c7825fa1bb884f326b619a879" = "sha256-xL0b6ZGmG4lhVcBjbBpobODZye6MAIr/gGBwMIrxmwM="; - "git+https://github.com/andrewgazelka/Flecs-Rust?rev=252944dedbc80741b7cca30dea67c5be95638950#252944dedbc80741b7cca30dea67c5be95638950" = - "sha256-3qUAXDHkeRFVfovZT+fW7VXW6aDteAiqCrLeCG/jd40="; + "git+https://github.com/andrewgazelka/Flecs-Rust?rev=f09dc5308d00c6a88c82b1195334b6ed2b2d2868#f09dc5308d00c6a88c82b1195334b6ed2b2d2868" = + "sha256-DlMOSY7NyoPoR8w4yswm3O97BegcNWcLl/fW3wOAmRs="; "git+https://github.com/TestingPlant/valence?branch=feat-bytes#fb792dcb6669b64c5dc2366eb3d074b293def046" = "sha256-rpuJSz8KxEwG5qeT4HYVtTxHJ24nrYZJwDurv+mjPxM="; };