Skip to content

Commit 381d27d

Browse files
mgmeiermgmeier
authored
Merge pull request IntersectMBO#5713 from IntersectMBO/andreabedini/genesis-creation-nix
[workbench] modular configuration for genesis
2 parents 04f37d5 + 794173b commit 381d27d

File tree

9 files changed

+666
-50
lines changed

9 files changed

+666
-50
lines changed

‎nix/workbench/evaluate/evaluate.sh‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# shellcheck shell=bash
2+
3+
# This is a small wrapper around nix-instantiate, which is exposed as `wb evaluate`.
4+
5+
usage_evaluate() {
6+
cat <<-EOF
7+
$(red USAGE:)
8+
9+
$(helpcmd evaluate \[FLAGS..] \[ATTRS...])
10+
11+
$(blue FLAGS):
12+
$(helpopt --profile-json PROFILE-JSON) Import the specified profile
13+
defaults to $(yellow \$WB_SHELL_PROFILE_DATA/profile.json) if set
14+
15+
$(helpopt --node-specs NODE-SPECS) Import the specified environment specification
16+
defaults to $(yellow \$WB_SHELL_PROFILE_DATA/node-specs.json) if set
17+
18+
$(helpopt --help) This help message
19+
20+
$(blue ATTRS):
21+
List of attributes to evaluate, defaults to the whole configuration.
22+
23+
Example:
24+
25+
wb evaluate evaluate genesis.create-staked-args
26+
EOF
27+
}
28+
29+
evaluate() {
30+
local profile_json node_specs
31+
local attrs=()
32+
local defaultAttrs=(--attr config)
33+
34+
while [ "$#" -gt 0 ]; do
35+
case "$1" in
36+
--)
37+
extra_args+=("$@")
38+
shift 1
39+
break
40+
;;
41+
--help)
42+
usage_evaluate
43+
exit 1
44+
;;
45+
--profile)
46+
profile_json=${2:?No value for argument $1}
47+
shift 2
48+
;;
49+
--node-specs)
50+
node_specs=${2:?No value for argument $1}
51+
shift 2
52+
;;
53+
*)
54+
attrs+=(--attr "config.$1")
55+
shift 1
56+
;;
57+
esac
58+
done
59+
60+
local err_missing_profile="You need to provide a profile: WB_SHELL_PROFILE_DATA is not set and at least one of --profile-json or --node-specs is missing."
61+
profile_json=${profile_json:-${WB_SHELL_PROFILE_DATA:?$err_missing_profile}/profile.json}
62+
node_specs=${node_specs:-${WB_SHELL_PROFILE_DATA:?$err_missing_profile}/node-specs.json}
63+
64+
nix-instantiate --eval --strict --json --no-warn-dirty --show-trace \
65+
--argstr profile-json "${profile_json}" \
66+
--argstr node-specs "${node_specs}" \
67+
"$WB_CARDANO_NODE_REPO_ROOT/nix/workbench/wb.nix" \
68+
"${attrs[@]:-${defaultAttrs[@]}}" \
69+
"${extra_args[@]}"
70+
}

‎nix/workbench/genesis/genesis.jq‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ def fmt_decimal_10_5($x):
55

66
def profile_cli_args($p):
77
{ common:
8-
{ createStackedArgs:
8+
{ createStakedArgs:
99
([ "--supply", fmt_decimal_10_5($p.genesis.funds_balance)
1010
, "--gen-utxo-keys", 1
1111
, "--gen-genesis-keys", $p.composition.n_bft_hosts

‎nix/workbench/genesis/genesis.sh‎

Lines changed: 145 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
global_genesis_format_version=March-14-2023
44

55
usage_genesis() {
6-
usage "genesis" "Genesis" <<EOF
6+
usage "genesis" "Genesis" <<EOF
77
$(helpcmd prepare-cache-entry [--force] PROFILE-JSON CACHEDIR NODE-SPECS OUTDIR)
88
Prepare a genesis cache entry for the specified profile.
99
Cache entry regeneration can be $(yellow --force)'d
@@ -84,7 +84,7 @@ case "$op" in
8484
local preset
8585

8686
if [[ -n "${regenesis_causes[*]}" ]]; then
87-
msg "genesis: generating due to ${regenesis_causes[*]}: $cache_key @$cache_path"
87+
info genesis "generating due to ${regenesis_causes[*]}: $cache_key @$cache_path"
8888

8989
jqtest .genesis.single_shot "$profile_json" ||
9090
fatal "Incremental (non single-shot) genesis is not supported."
@@ -109,13 +109,15 @@ case "$op" in
109109
version=$(cat "$cache_dir"/layout.version 2>/dev/null || echo "unknown")
110110

111111
if [[ ! "$version" == "$global_genesis_format_version" ]]; then
112-
msg "genesis: cache entry at $cache_dir is incompatible: layout version '$version' does not match current: $global_genesis_format_version"
112+
info genesis "cache entry at $cache_dir is incompatible: layout version '$version' does not match current: $global_genesis_format_version"
113113
return 1;
114114
fi
115115
return 0
116116
;;
117117

118118
profile-cache-key-input )
119+
[[ "$WB_MODULAR_GENESIS" -eq 1 ]] && { genesis "$op-modular" "$@"; return; }
120+
119121
local usage="USAGE: wb genesis $op PROFILE-JSON"
120122
local profile_json=${1:?$usage}
121123

@@ -133,7 +135,22 @@ case "$op" in
133135
' "${args[@]}"
134136
;;
135137

138+
profile-cache-key-input-modular )
139+
info genesis "profile-cache-key-input ($(red using modular configuration))"
140+
141+
local usage="USAGE: wb genesis $op PROFILE-JSON"
142+
local profile_json=${1:?$usage}
143+
144+
# nix wants absolute paths
145+
profile_json=$(realpath "$profile_json")
146+
147+
# NOTE: jq is only used for formatting
148+
evaluate --profile "${profile_json}" genesis.cache-key-input | jq
149+
;;
150+
136151
profile-cache-key )
152+
[[ "$WB_MODULAR_GENESIS" -eq 1 ]] && { genesis "$op-modular" "$@"; return; }
153+
137154
local usage="USAGE: wb genesis $op PROFILE-JSON"
138155
local profile_json=${1:?$usage}
139156

@@ -145,17 +162,31 @@ case "$op" in
145162
-L "$global_basedir/genesis"
146163
)
147164
jq 'include "genesis";
148-
149165
profile_genesis_cache_entry_name($profile[0]; $params_hash)
150166
' "${args[@]}" "$profile_json"
151167
;;
152168

169+
profile-cache-key-modular )
170+
info genesis "profile-cache-key ($(red using modular configuration))"
171+
172+
local usage="USAGE: wb genesis $op PROFILE-JSON"
173+
local profile_json=${1:?$usage}
174+
175+
# nix wants absolute paths
176+
profile_json=$(realpath "$profile_json")
177+
178+
# NOTE:
179+
# - the hash is different because nix cannot reproduce jq's pretty-printing
180+
# - jq is only used for formatting
181+
evaluate --profile "${profile_json}" genesis.cache-key | jq -r
182+
;;
183+
153184
genesis-from-preset )
154185
local usage="USAGE: wb genesis $op PRESET GENESIS-DIR"
155186
local preset=${1:?$usage}
156187
local dir=${2:?$usage}
157188

158-
msg "genesis: profile uses preset genesis: $preset"
189+
info genesis "profile uses preset genesis: $preset"
159190

160191
rm -rf "$dir"/{*-keys,byron,pools,nodes,*.json,*.params,*.version}
161192
mkdir -p "$dir/byron"
@@ -170,6 +201,7 @@ case "$op" in
170201

171202
actually-genesis )
172203
set -euo pipefail
204+
[[ "$WB_MODULAR_GENESIS" -eq 1 ]] && { genesis "$op-modular" "$@"; return; }
173205

174206
local usage="USAGE: wb genesis $op PROFILE-JSON NODE-SPECS DIR"
175207
local profile_json=${1:-$WB_SHELL_PROFILE_DATA/profile.json}
@@ -220,7 +252,58 @@ case "$op" in
220252
| from_entries
221253
' "$node_specs" > "$dir"/pool-relays.json
222254

223-
read -r -a args <<< "$(jq --raw-output '.cli_args.createStackedArgs | join(" ")' "$profile_json")"
255+
read -r -a args <<< "$(jq --raw-output '.cli_args.createStakedArgs | join(" ")' "$profile_json")"
256+
create_staked_args=(
257+
--genesis-dir "$dir"
258+
--relay-specification-file "$dir/pool-relays.json"
259+
"${args[@]}"
260+
)
261+
verbose "genesis" "$(colorise cardano-cli genesis create-staked "${create_staked_args[@]}")"
262+
cardano-cli genesis create-staked "${create_staked_args[@]}"
263+
mv "$dir"/genesis.json "$dir"/genesis-shelley.json
264+
mv "$dir"/genesis.spec.json "$dir"/genesis-shelley.spec.json
265+
266+
info genesis "removing delegator keys.."
267+
rm "$dir"/stake-delegator-keys -rf
268+
269+
info genesis "moving keys"
270+
Massage_the_key_file_layout_to_match_AWS "$profile_json" "$node_specs" "$dir"
271+
272+
info genesis "sealing"
273+
cat <<<"$cache_key_input" > "$dir"/cache.key.input
274+
cat <<<"$cache_key" > "$dir"/cache.key
275+
cat <<<"$global_genesis_format_version" > "$dir"/layout.version
276+
;;
277+
278+
actually-genesis-modular )
279+
set -euo pipefail
280+
info genesis "actually-genesis ($(red using modular configuration))"
281+
282+
local usage="USAGE: wb genesis $op PROFILE-JSON NODE-SPECS DIR"
283+
local profile_json=${1:-$WB_SHELL_PROFILE_DATA/profile.json}
284+
local node_specs=${2:-$WB_SHELL_PROFILE_DATA/node-specs.json}
285+
local dir=${3:-$(mktemp -d)}
286+
local cache_key_input=${4:-$(genesis profile-cache-key-input "$WB_SHELL_PROFILE_DATA"/profile.json)}
287+
local cache_key=${5:-$(genesis profile-cache-key "$WB_SHELL_PROFILE_DATA"/profile.json)}
288+
289+
progress "genesis" "new one: $(yellow profile) $(blue "$profile_json") $(yellow node_specs) $(blue "$node_specs") $(yellow dir) $(blue "$dir") $(yellow cache_key) $(blue "$cache_key") $(yellow cache_key_input) $(blue "$cache_key_input")"
290+
291+
rm -rf "$dir"/{*-keys,byron,pools,nodes,*.json,*.params,*.version}
292+
mkdir -p "$dir"
293+
294+
# nix wants absolute paths
295+
profile_json=$(realpath "$profile_json")
296+
node_specs=$(realpath "$node_specs")
297+
298+
local EVAL="evaluate --profile ${profile_json} --node-specs ${node_specs}"
299+
300+
# NOTE: jq is only used for formatting
301+
$EVAL genesis.shelley | jq > "$dir/genesis.spec.json"
302+
$EVAL genesis.alonzo | jq > "$dir/genesis.alonzo.spec.json"
303+
$EVAL genesis.conway | jq > "$dir/genesis.conway.spec.json"
304+
$EVAL genesis.pool-relays | jq > "$dir/pool-relays.json"
305+
306+
read -r -a args <<< "$($EVAL genesis.create-staked-args | jq -r)"
224307
create_staked_args=(
225308
--genesis-dir "$dir"
226309
--relay-specification-file "$dir/pool-relays.json"
@@ -231,17 +314,17 @@ case "$op" in
231314
mv "$dir"/genesis.json "$dir"/genesis-shelley.json
232315
mv "$dir"/genesis.spec.json "$dir"/genesis-shelley.spec.json
233316

234-
msg "genesis: removing delegator keys.."
317+
info genesis "removing delegator keys.."
235318
rm "$dir"/stake-delegator-keys -rf
236319

237-
msg "genesis: moving keys"
238-
## TODO: try to get rid of this step:
320+
info genesis "moving keys"
239321
Massage_the_key_file_layout_to_match_AWS "$profile_json" "$node_specs" "$dir"
240322

241-
msg "genesis: sealing"
323+
info genesis "sealing"
242324
cat <<<"$cache_key_input" > "$dir"/cache.key.input
243325
cat <<<"$cache_key" > "$dir"/cache.key
244-
cat <<<"$global_genesis_format_version" > "$dir"/layout.version;;
326+
cat <<<"$global_genesis_format_version" > "$dir"/layout.version
327+
;;
245328

246329
derive-from-cache )
247330
local usage="USAGE: wb genesis $op PROFILE-OUT TIMING-JSON-EXPR CACHE-ENTRY-DIR OUTDIR"
@@ -289,6 +372,8 @@ case "$op" in
289372
;;
290373

291374
finalise-cache-entry )
375+
[[ "$WB_MODULAR_GENESIS" -eq 1 ]] && { genesis "$op-modular" "$@"; return; }
376+
292377
local usage="USAGE: wb genesis $op PROFILE-JSON TIMING-JSON-EXPR DIR"
293378
local profile_json=${1:?$usage}
294379
local timing=${2:?$usage}
@@ -307,7 +392,51 @@ case "$op" in
307392
sponge "$dir"/genesis-shelley.json
308393
;;
309394

310-
* ) usage_genesis;; esac
395+
finalise-cache-entry-modular )
396+
set -euo pipefail
397+
info genesis "finalise-cache-entry ($(red using modular configuration))"
398+
399+
local usage="USAGE: wb genesis $op PROFILE-JSON TIMING-JSON-EXPR DIR"
400+
local profile_json=${1:?$usage}
401+
local timing=${2:?$usage}
402+
local dir=${3:?$usage}
403+
404+
if profile has-preset "$profile_json"; then
405+
return
406+
fi
407+
408+
progress "genesis" "deriving from cache: $cache_entry -> $outdir"
409+
410+
local system_start_epoch
411+
system_start_epoch="$(jq '.start' -r <<<"$timing")"
412+
413+
# nix wants absolute paths
414+
profile_json=$(realpath "$profile_json")
415+
416+
evaluate --profile "$profile_json" genesis.byron > "$dir"/byron-protocol-params.json
417+
read -r -a args <<< "$(evaluate --profile "${profile_json}" genesis.byron-genesis-args | jq -r)"
418+
419+
cli_args=(
420+
--genesis-output-dir "$dir"/byron
421+
--protocol-parameters-file "$dir"/byron-protocol-params.json
422+
--start-time "$system_start_epoch"
423+
"${args[@]}"
424+
)
425+
rm -rf "$dir"/byron
426+
427+
verbose "genesis" "$(colorise cardano-cli byron genesis genesis "${cli_args[@]}")"
428+
cardano-cli byron genesis genesis "${cli_args[@]}"
429+
430+
# We need to change systemStart in genesis-shelley to make it compatible with system_start_epoch
431+
jq '. * { systemStart: $timing.systemStart }' --argjson timing "$timing" \
432+
"$dir"/genesis-shelley.json | sponge "$dir"/genesis-shelley.json
433+
;;
434+
435+
* )
436+
usage_genesis
437+
;;
438+
439+
esac
311440
}
312441

313442
__KEY_ROOT=
@@ -322,7 +451,7 @@ Massage_the_key_file_layout_to_match_AWS() {
322451
if [[ -z "$pool_density_map" ]]; then
323452
fatal "failed: topology density-map '$node_specs'"
324453
fi
325-
msg "genesis: pool density map: $pool_density_map"
454+
info genesis "pool density map: $pool_density_map"
326455

327456
__KEY_ROOT=$dir
328457

@@ -336,12 +465,12 @@ Massage_the_key_file_layout_to_match_AWS() {
336465
if jqtest ".genesis.dense_pool_density > 1" "$profile_json" &&
337466
jqtest ".[\"$id\"] > 1" <<<"$pool_density_map"
338467
then ## Dense/bulk pool
339-
msg "genesis: bulk pool $did -> node-$id"
468+
info genesis "bulk pool $did -> node-$id"
340469
cp -f "$(key_genesis bulk bulk "$did")" "$(key_depl bulk bulk "$id")"
341470
did=$((did + 1))
342471
elif jqtest ".[\"$id\"] != 0" <<<"$pool_density_map"
343472
then ## Singular pool
344-
msg "genesis: pool $pid -> node-$id"
473+
info genesis "pool $pid -> node-$id"
345474
cp -f "$(key_genesis cold sig "$pid")" "$(key_depl cold sig "$id")"
346475
cp -f "$(key_genesis cold ver "$pid")" "$(key_depl cold ver "$id")"
347476
cp -f "$(key_genesis opcert cert "$pid")" "$(key_depl opcert none "$id")"
@@ -352,7 +481,7 @@ Massage_the_key_file_layout_to_match_AWS() {
352481
cp -f "$(key_genesis VRF ver "$pid")" "$(key_depl VRF ver "$id")"
353482
pid=$((pid + 1))
354483
else ## BFT node
355-
msg "genesis: BFT $bid -> node-$id"
484+
info genesis "BFT $bid -> node-$id"
356485
cp -f "$(key_genesis deleg sig "$bid")" "$(key_depl cold sig "$id")"
357486
cp -f "$(key_genesis deleg ver "$bid")" "$(key_depl cold ver "$id")"
358487
cp -f "$(key_genesis delegCert cert "$bid")" "$(key_depl opcert none "$id")"

‎nix/workbench/lib.sh‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,12 +179,21 @@ red() {
179179
eval $restore_trace
180180
}
181181

182+
info() {
183+
eval $muffle_trace_set_exit
184+
185+
local subsys=$1; shift
186+
msg "$(with_color cyan "$subsys"): $*"
187+
188+
eval $restore_trace
189+
}
190+
182191
verbose() {
183192
eval $muffle_trace_set_exit
184193

185194
if test -n "${verbose:-}"
186195
then local subsys=$1; shift
187-
msg "$(with_color blue $subsys): $*"
196+
msg "$(with_color blue "$subsys"): $*"
188197
fi
189198

190199
eval $restore_trace
@@ -194,7 +203,7 @@ progress() {
194203
eval $muffle_trace_set_exit
195204

196205
local subsys=$1; shift
197-
msg "$(with_color green $subsys): $(with_color blue "$@")"
206+
msg "$(with_color green "$subsys"): $(with_color blue "$@")"
198207

199208
eval $restore_trace
200209
}

0 commit comments

Comments
 (0)