Skip to content

Commit 947aa24

Browse files
vinnybodMarconZet
andauthored
V3 lockfiles (#69)
* Added non-conflicting hash for install files * cherry-pick vinnybod/fix-issue-1498 --------- Co-authored-by: MarconZet <[email protected]>
2 parents c142040 + 82d27b2 commit 947aa24

24 files changed

+1150
-112
lines changed

BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ exports_files([
66
"defs.bzl",
77
"extensions.bzl",
88
"specs.bzl",
9+
"maven_install.json",
910
])
1011

1112
licenses(["notice"]) # Apache 2.0

maven_install.json

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,31 @@
11
{
22
"__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL",
3-
"__INPUT_ARTIFACTS_HASH": 1448912684,
4-
"__RESOLVED_ARTIFACTS_HASH": 771049556,
3+
"__INPUT_ARTIFACTS_HASH": {
4+
"com.google.guava:guava": 1376266919,
5+
"com.google.protobuf:protobuf-java": -414274684,
6+
"io.netty:netty-tcnative-boringssl-static": -1979050407,
7+
"org.hamcrest:hamcrest-core": 466791695,
8+
"repositories": -1949687017
9+
},
10+
"__RESOLVED_ARTIFACTS_HASH": {
11+
"com.google.code.findbugs:jsr305": 870839855,
12+
"com.google.errorprone:error_prone_annotations": -924787181,
13+
"com.google.guava:failureaccess": -1890754729,
14+
"com.google.guava:guava": 1410177884,
15+
"com.google.guava:listenablefuture": 1079558157,
16+
"com.google.j2objc:j2objc-annotations": 880287147,
17+
"com.google.protobuf:protobuf-java": -758252690,
18+
"io.netty:netty-tcnative-boringssl-static": 786460467,
19+
"io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64": -151974322,
20+
"io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64": -1831640381,
21+
"io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64": -1661340718,
22+
"io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64": 2101324017,
23+
"io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64": 889950966,
24+
"io.netty:netty-tcnative-classes": 1239547355,
25+
"org.checkerframework:checker-qual": -1034954841,
26+
"org.hamcrest:hamcrest": -533823501,
27+
"org.hamcrest:hamcrest-core": 511008887
28+
},
529
"artifacts": {
630
"com.google.code.findbugs:jsr305": {
731
"shasums": {
@@ -250,5 +274,5 @@
250274
]
251275
},
252276
"services": {},
253-
"version": "2"
277+
"version": "3"
254278
}

private/extensions/maven.bzl

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ load("//private/lib:toml_parser.bzl", "parse_toml")
1313
load("//private/rules:coursier.bzl", "DEFAULT_AAR_IMPORT_LABEL", "coursier_fetch", "pinned_coursier_fetch")
1414
load("//private/rules:unpinned_maven_pin_command_alias.bzl", "unpinned_maven_pin_command_alias")
1515
load("//private/rules:v1_lock_file.bzl", "v1_lock_file")
16-
load("//private/rules:v2_lock_file.bzl", "v2_lock_file")
16+
load("//private/rules:v3_lock_file.bzl", "v2_lock_file", "v3_lock_file")
1717
load(":download_pinned_deps.bzl", "download_pinned_deps")
1818

1919
DEFAULT_REPOSITORIES = [
@@ -726,12 +726,15 @@ def maven_impl(mctx):
726726
"artifacts": {},
727727
"dependencies": {},
728728
"repositories": {},
729-
"version": "2",
729+
"version": "3",
730730
}
731731
else:
732732
lock_file = json.decode(lock_file_content)
733733

734-
if v2_lock_file.is_valid_lock_file(lock_file):
734+
if v3_lock_file.is_valid_lock_file(lock_file):
735+
artifacts = v3_lock_file.get_artifacts(lock_file)
736+
importer = v3_lock_file
737+
elif v2_lock_file.is_valid_lock_file(lock_file):
735738
artifacts = v2_lock_file.get_artifacts(lock_file)
736739
importer = v2_lock_file
737740
elif v1_lock_file.is_valid_lock_file(lock_file):

private/rules/coursier.bzl

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ load(
3131
)
3232
load("//private/lib:urls.bzl", "remove_auth_from_url")
3333
load("//private/rules:v1_lock_file.bzl", "v1_lock_file")
34-
load("//private/rules:v2_lock_file.bzl", "v2_lock_file")
34+
load("//private/rules:v3_lock_file.bzl", "v2_lock_file", "v3_lock_file")
3535

3636
_BUILD = """
3737
# package(default_visibility = [{visibilities}]) # https://github.com/bazelbuild/bazel/issues/13681
@@ -328,6 +328,13 @@ def _stable_artifact(artifact):
328328
keys = sorted(parsed.keys())
329329
return ":".join(["%s=%s" % (key, parsed[key]) for key in keys])
330330

331+
def _add_to_hash_dictionary(dictionary, artifact, salt):
332+
artifact_dict = json.decode(artifact)
333+
key = artifact_dict["group"] + ":" + artifact_dict["artifact"]
334+
value = dictionary.get(key, [])
335+
value.append(hash(_stable_artifact(artifact) + salt))
336+
dictionary[key] = value
337+
331338
# Compute a signature of the list of artifacts that will be used to build
332339
# the dependency tree. This is used as a check to see whether the dependency
333340
# tree needs to be repinned.
@@ -346,24 +353,37 @@ def compute_dependency_inputs_signature(boms = [], artifacts = [], repositories
346353
artifact_inputs = []
347354
excluded_artifact_inputs = []
348355

356+
all_hashes = dict()
357+
349358
if boms and len(boms):
350359
for bom in sorted(boms):
351360
artifact_inputs.append(_stable_artifact(bom))
361+
_add_to_hash_dictionary(all_hashes, bom, "bom")
352362

353363
for artifact in sorted(artifacts):
354364
artifact_inputs.append(_stable_artifact(artifact))
365+
_add_to_hash_dictionary(all_hashes, artifact, "artifact")
355366

356367
for artifact in sorted(excluded_artifacts):
357368
excluded_artifact_inputs.append(_stable_artifact(artifact))
369+
_add_to_hash_dictionary(all_hashes, artifact, "excluded_artifact")
358370

359371
v1_sig = hash(repr(sorted(artifact_inputs))) ^ hash(repr(sorted(repositories)))
360372

361373
hash_parts = [sorted(artifact_inputs), sorted(repositories), sorted(excluded_artifact_inputs)]
362-
current_version_sig = 0
374+
v2_sig = 0
363375
for part in hash_parts:
364-
current_version_sig ^= hash(repr(part))
376+
v2_sig ^= hash(repr(part))
377+
378+
for k, v in all_hashes.items():
379+
if len(v) == 1:
380+
all_hashes[k] = v[0]
381+
else:
382+
all_hashes[k] = hash(repr(sorted(v)))
365383

366-
return (current_version_sig, [v1_sig])
384+
all_hashes["repositories"] = hash(repr(sorted(repositories)))
385+
386+
return (all_hashes, [v1_sig, v2_sig])
367387

368388
def get_netrc_lines_from_entries(netrc_entries):
369389
netrc_lines = []
@@ -460,21 +480,26 @@ def _pinned_coursier_fetch_impl(repository_ctx):
460480
"artifacts": {},
461481
"dependencies": {},
462482
"repositories": {},
463-
"version": "2",
483+
"version": "3",
464484
}
465485
else:
466486
maven_install_json_content = json.decode(lock_file_content)
467487

468-
if v1_lock_file.is_valid_lock_file(maven_install_json_content):
488+
if v3_lock_file.is_valid_lock_file(maven_install_json_content):
489+
importer = v3_lock_file
490+
elif v2_lock_file.is_valid_lock_file(maven_install_json_content):
491+
importer = v2_lock_file
492+
elif v1_lock_file.is_valid_lock_file(maven_install_json_content):
469493
importer = v1_lock_file
494+
else:
495+
fail("Unable to read lock file: %s" % repository_ctx.attr.maven_install_json)
496+
497+
# Check if using the most recent lock file format.
498+
if importer != v3_lock_file:
470499
print_if_not_repinning(
471500
repository_ctx,
472501
"Lock file should be updated. Please run `REPIN=1 bazel run @unpinned_%s//:pin`" % repository_ctx.name,
473502
)
474-
elif v2_lock_file.is_valid_lock_file(maven_install_json_content):
475-
importer = v2_lock_file
476-
else:
477-
fail("Unable to read lock file: %s" % repository_ctx.attr.maven_install_json)
478503

479504
# Validation steps for maven_install.json.
480505

@@ -1306,7 +1331,7 @@ def _coursier_fetch_impl(repository_ctx):
13061331

13071332
repository_ctx.file(
13081333
"unsorted_deps.json",
1309-
content = v2_lock_file.render_lock_file(
1334+
content = v3_lock_file.render_lock_file(
13101335
lock_file_contents,
13111336
inputs_hash,
13121337
),
@@ -1315,7 +1340,7 @@ def _coursier_fetch_impl(repository_ctx):
13151340
repository_ctx.report_progress("Generating BUILD targets..")
13161341
(generated_imports, jar_versionless_target_labels) = parser.generate_imports(
13171342
repository_ctx = repository_ctx,
1318-
dependencies = v2_lock_file.get_artifacts(lock_file_contents),
1343+
dependencies = v3_lock_file.get_artifacts(lock_file_contents),
13191344
explicit_artifacts = {
13201345
a["group"] + ":" + a["artifact"] + (":" + a["classifier"] if "classifier" in a else ""): True
13211346
for a in artifacts

private/rules/maven_utils.bzl

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,6 @@ def generate_pom(
101101
"{classifier}": unpacked_coordinates.classifier or "jar",
102102
}
103103

104-
for key in exclusions:
105-
if key not in versioned_dep_coordinates and key not in unversioned_dep_coordinates:
106-
fail("Key %s in exclusions does not occur in versioned_dep_coordinates or unversioned_dep_coordinates" % key)
107-
108104
if parent:
109105
# We only want the groupId, artifactID, and version
110106
unpacked_parent = _unpack_coordinates(parent)

private/rules/pin_dependencies.bzl

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ load("//private/rules:coursier.bzl", "compute_dependency_inputs_signature")
1515

1616
_TEMPLATE = """#!/usr/bin/env bash
1717
18-
{resolver_cmd} --jvm_flags={jvm_flags} --argsfile {config} --input_hash '{input_hash}' --output {output}
18+
{resolver_cmd} --jvm_flags={jvm_flags} --argsfile {config} --input-hash-path '{input_hash_path}' --output {output}
1919
"""
2020

2121
def _stringify_exclusions(exclusions):
@@ -63,19 +63,25 @@ def _pin_dependencies_impl(ctx):
6363
content = json.encode_indent(config, indent = " "),
6464
)
6565

66-
input_hash = compute_dependency_inputs_signature(
66+
input_hash, _ = compute_dependency_inputs_signature(
6767
boms = ctx.attr.boms,
6868
artifacts = ctx.attr.artifacts,
6969
repositories = ctx.attr.repositories,
7070
excluded_artifacts = ctx.attr.excluded_artifacts,
7171
)
7272

73+
hash_file = ctx.actions.declare_file("%s-input-hash.json" % ctx.label.name)
74+
ctx.actions.write(
75+
hash_file,
76+
content = json.encode_indent(input_hash, indent = " "),
77+
)
78+
7379
script = ctx.actions.declare_file(ctx.label.name)
7480
ctx.actions.write(
7581
script,
7682
_TEMPLATE.format(
7783
config = config_file.short_path,
78-
input_hash = input_hash[0],
84+
input_hash_path = hash_file.short_path,
7985
resolver_cmd = ctx.executable.resolver.short_path,
8086
output = "$BUILD_WORKSPACE_DIRECTORY/" + ctx.attr.lock_file,
8187
jvm_flags = ctx.attr.jvm_flags,
@@ -87,7 +93,7 @@ def _pin_dependencies_impl(ctx):
8793
DefaultInfo(
8894
executable = script,
8995
files = depset([script, config_file]),
90-
runfiles = ctx.runfiles(files = [script, config_file]).merge(ctx.attr.resolver[DefaultInfo].default_runfiles),
96+
runfiles = ctx.runfiles(files = [script, config_file, hash_file]).merge(ctx.attr.resolver[DefaultInfo].default_runfiles),
9197
),
9298
]
9399

Lines changed: 93 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,15 @@ load("//private/lib:coordinates.bzl", "to_external_form")
1515

1616
_REQUIRED_KEYS = ["artifacts", "dependencies", "repositories"]
1717

18-
def _is_valid_lock_file(lock_file_contents):
18+
def _is_valid_lock_file_v2(lock_file_contents):
19+
return _is_valid_lock_file(lock_file_contents, "2")
20+
21+
def _is_valid_lock_file_v3(lock_file_contents):
22+
return _is_valid_lock_file(lock_file_contents, "3")
23+
24+
def _is_valid_lock_file(lock_file_contents, desired_version):
1925
version = lock_file_contents.get("version")
20-
if "2" != version:
26+
if desired_version != version:
2127
return False
2228

2329
all_keys = lock_file_contents.keys()
@@ -37,7 +43,7 @@ def _get_input_artifacts_hash(lock_file_contents):
3743
def _get_lock_file_hash(lock_file_contents):
3844
return lock_file_contents.get("__RESOLVED_ARTIFACTS_HASH")
3945

40-
def _compute_lock_file_hash(lock_file_contents):
46+
def _compute_lock_file_hash_v2(lock_file_contents):
4147
to_hash = {}
4248
for key in sorted(_REQUIRED_KEYS):
4349
value = lock_file_contents.get(key)
@@ -47,6 +53,75 @@ def _compute_lock_file_hash(lock_file_contents):
4753
to_hash.update({key: json.decode(json.encode(value))})
4854
return hash(repr(to_hash))
4955

56+
def _compute_final_hash(all_infos):
57+
final_hashes = dict()
58+
59+
# in case of circular dependencies, we take a normal hash of the original info as a starting point
60+
backup_hashes = {k: hash(repr(v)) for k, v in all_infos.items()}
61+
62+
# sets are balzel 8 only, we use a dict instead
63+
remaining = {k: 0 for k in all_infos.keys()}
64+
65+
# bazel does not support recursion, we have to emulate it manually
66+
stack = []
67+
68+
# replacement for while True
69+
for _ in range(1000000000):
70+
if len(remaining) == 0 and len(stack) == 0:
71+
break
72+
73+
curr = None
74+
if len(stack) == 0:
75+
curr, _ = remaining.popitem()
76+
else:
77+
curr = stack.pop()
78+
79+
if curr in final_hashes:
80+
continue
81+
82+
deps = all_infos[curr].get("dependencies", [])
83+
84+
# make sure all dependencies are processed first
85+
unprocessed = [d for d in deps if d in remaining]
86+
if len(unprocessed) > 0:
87+
dep = unprocessed[0]
88+
stack.append(curr)
89+
stack.append(dep)
90+
remaining.pop(dep, None)
91+
continue
92+
93+
all_infos[curr]["dependency_hashes"] = {dep: final_hashes.get(dep, backup_hashes.get(dep, 0)) for dep in deps}
94+
final_hashes[curr] = hash(repr(all_infos[curr]))
95+
96+
return final_hashes
97+
98+
def _compute_lock_file_hash_v3(lock_file_contents):
99+
all_infos = dict()
100+
101+
for dep, dep_info in lock_file_contents["artifacts"].items():
102+
shasums = dep_info["shasums"]
103+
common_info = {k: v for k, v in dep_info.items() if k != "shasums"}
104+
105+
is_jar_type = dep.count(":") == 1
106+
107+
for type, sha in shasums.items():
108+
jar_suffix = ":jar" if is_jar_type else ""
109+
suffix = jar_suffix + ":" + type if type != "jar" else ""
110+
111+
type_info = dict()
112+
type_info["standard"] = common_info
113+
type_info["sha"] = sha
114+
all_infos[dep + suffix] = type_info
115+
116+
for repo, artifacts in lock_file_contents["repositories"].items():
117+
for artifact in artifacts:
118+
all_infos[artifact]["repository"] = repo
119+
120+
for dep, dep_info in lock_file_contents["dependencies"].items():
121+
all_infos[dep]["dependencies"] = sorted(dep_info)
122+
123+
return _compute_final_hash(all_infos)
124+
50125
def _to_m2_path(unpacked):
51126
path = "{group}/{artifact}/{version}/{artifact}-{version}".format(
52127
artifact = unpacked["artifact"],
@@ -188,8 +263,8 @@ def _render_lock_file(lock_file_contents, input_hash):
188263
contents = [
189264
"{",
190265
" \"__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY\": \"THERE_IS_NO_DATA_ONLY_ZUUL\",",
191-
" \"__INPUT_ARTIFACTS_HASH\": %s," % input_hash,
192-
" \"__RESOLVED_ARTIFACTS_HASH\": %s," % _compute_lock_file_hash(lock_file_contents),
266+
" \"__INPUT_ARTIFACTS_HASH\": %s," % json.encode_indent(input_hash, prefix = " ", indent = " "),
267+
" \"__RESOLVED_ARTIFACTS_HASH\": %s," % json.encode_indent(_compute_lock_file_hash_v3(lock_file_contents), prefix = " ", indent = " "),
193268
]
194269
if lock_file_contents.get("conflict_resolution"):
195270
contents.append(" \"conflict_resolution\": %s," % json.encode_indent(lock_file_contents["conflict_resolution"], prefix = " ", indent = " "))
@@ -212,17 +287,27 @@ def _render_lock_file(lock_file_contents, input_hash):
212287
contents.append(" \"services\": %s," % json.encode_indent(lock_file_contents["services"], prefix = " ", indent = " "))
213288
if lock_file_contents.get("skipped"):
214289
contents.append(" \"skipped\": %s," % json.encode_indent(lock_file_contents["skipped"], prefix = " ", indent = " "))
215-
contents.append(" \"version\": \"2\"")
290+
contents.append(" \"version\": \"3\"")
216291
contents.append("}")
217292
contents.append("")
218293

219294
return "\n".join(contents)
220295

221296
v2_lock_file = struct(
222-
is_valid_lock_file = _is_valid_lock_file,
297+
is_valid_lock_file = _is_valid_lock_file_v2,
298+
get_input_artifacts_hash = _get_input_artifacts_hash,
299+
get_lock_file_hash = _get_lock_file_hash,
300+
compute_lock_file_hash = _compute_lock_file_hash_v2,
301+
get_artifacts = _get_artifacts,
302+
get_netrc_entries = _get_netrc_entries,
303+
has_m2local = _has_m2local,
304+
)
305+
306+
v3_lock_file = struct(
307+
is_valid_lock_file = _is_valid_lock_file_v3,
223308
get_input_artifacts_hash = _get_input_artifacts_hash,
224309
get_lock_file_hash = _get_lock_file_hash,
225-
compute_lock_file_hash = _compute_lock_file_hash,
310+
compute_lock_file_hash = _compute_lock_file_hash_v3,
226311
get_artifacts = _get_artifacts,
227312
get_netrc_entries = _get_netrc_entries,
228313
render_lock_file = _render_lock_file,

0 commit comments

Comments
 (0)