diff --git a/fusesoc/coremanager.py b/fusesoc/coremanager.py index 4666d306..a69f5aa2 100644 --- a/fusesoc/coremanager.py +++ b/fusesoc/coremanager.py @@ -4,6 +4,7 @@ import logging import os +import pathlib from okonomiyaki.versions import EnpkgVersion from simplesat.constraints import PrettyPackageStringParser, Requirement @@ -16,6 +17,8 @@ from fusesoc.capi2.coreparser import Core2Parser from fusesoc.core import Core from fusesoc.librarymanager import LibraryManager +from fusesoc.lockfile import load_lockfile +from fusesoc.vlnv import Vlnv, compare_relation logger = logging.getLogger(__name__) @@ -33,6 +36,7 @@ class CoreDB: def __init__(self): self._cores = {} self._solver_cache = {} + self._lockfile = None # simplesat doesn't allow ':', '-' or leading '_' def _package_name(self, vlnv): @@ -45,6 +49,7 @@ def _package_version(self, vlnv): def _parse_depend(self, depends): # FIXME: Handle conflicts deps = [] + _s = "{} {} {}" for d in depends: for simple in d.simpleVLNVs(): @@ -83,6 +88,9 @@ def find(self, vlnv=None): found = list([core["core"] for core in self._cores.values()]) return found + def load_lockfile(self, filepath: pathlib.Path): + self._lockfile = load_lockfile(filepath) + def _solver_cache_lookup(self, key): if key in self._solver_cache: return self._solver_cache[key] @@ -110,6 +118,27 @@ def _hash_flags_dict(self, flags): h ^= hash(pair) return h + def _lockfile_replace(self, core: Vlnv): + """Try to pin the core version from cores defined in the lock file""" + if self._lockfile: + for locked_core in self._lockfile["cores"]: + if locked_core.vln_str() == core.vln_str(): + valid_version = compare_relation(locked_core, core.relation, core) + if valid_version: + core.version = locked_core.version + core.revision = locked_core.revision + core.relation = "==" + else: + # Invalid version in lockfile + logger.warning( + "Failed to pin core {} outside of dependency version {} {} {}".format( + str(locked_core), + core.vln_str(), + core.relation, + core.version, + ) + ) + def solve(self, top_core, flags): return self._solve(top_core, flags) @@ -195,8 +224,12 @@ def eq_vln(this, that): _flags["is_toplevel"] = core.name == top_core _depends = core.get_depends(_flags) if _depends: + for depend in _depends: + self._lockfile_replace(depend) _s = "; depends ( {} )" package_str += _s.format(self._parse_depend(_depends)) + else: + self._lockfile_replace(top_core) parser = PrettyPackageStringParser(EnpkgVersion.from_string) @@ -226,6 +259,7 @@ def eq_vln(this, that): raise DependencyError(top_core.name) virtual_selection = {} + partial_lockfile = False objdict = {} if len(transaction.operations) > 1: for op in transaction.operations: @@ -244,6 +278,11 @@ def eq_vln(this, that): if p[0] in virtual_selection: # If package that implements a virtual core is required, remove from the dictionary del virtual_selection[p[0]] + if ( + self._lockfile + and op.package.core.name not in self._lockfile["cores"] + ): + partial_lockfile = True op.package.core.direct_deps = [ objdict[n[0]] for n in op.package.install_requires ] @@ -254,6 +293,8 @@ def eq_vln(this, that): virtual[1], virtual[0] ) ) + if partial_lockfile: + logger.warning("Using lock file with partial list of cores") result = [op.package.core for op in transaction.operations] diff --git a/fusesoc/lockfile.py b/fusesoc/lockfile.py new file mode 100644 index 00000000..6aa0a104 --- /dev/null +++ b/fusesoc/lockfile.py @@ -0,0 +1,76 @@ +import json +import logging +import os +import pathlib + +import fastjsonschema + +import fusesoc.utils +from fusesoc.version import version +from fusesoc.vlnv import Vlnv + +logger = logging.getLogger(__name__) + +lockfile_schema = """ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "FuseSoC Lockfile", + "description": "FuseSoC Lockfile", + "type": "object", + "properties": { + "cores": { + "description": "Cores used in the build", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "description": "Core VLVN", + "type": "string" + } + } + } + }, + "fusesoc_version": { + "description": "FuseSoC version which generated the lockfile", + "type": "string" + }, + "lockfile_version": { + "description": "Lockfile version", + "type": "integer" + } + } +} +""" + + +def load_lockfile(filepath: pathlib.Path): + try: + lockfile_data = fusesoc.utils.yaml_fread(filepath) + try: + validator = fastjsonschema.compile( + json.loads(lockfile_schema), detailed_exceptions=False + ) + validator(lockfile_data) + except fastjsonschema.JsonSchemaDefinitionException as e: + raise SyntaxError(f"Error parsing JSON Schema: {e}") + except fastjsonschema.JsonSchemaException as e: + raise SyntaxError(f"Error validating {e}") + except FileNotFoundError: + logger.warning(f"Lockfile {filepath} not found") + return None + + cores = {} + for core in lockfile_data.setdefault("cores", []): + if "name" in core: + vlnv = Vlnv(core["name"]) + vln = vlnv.vln_str() + if vln in map(Vlnv.vln_str, cores.keys()): + raise SyntaxError(f"Core {vln} defined multiple times in lock file") + cores[vlnv] = {"name": vlnv} + else: + raise SyntaxError(f"Core definition without a name") + lockfile = { + "cores": cores, + } + return lockfile diff --git a/fusesoc/main.py b/fusesoc/main.py index 47c7fb30..8838437e 100644 --- a/fusesoc/main.py +++ b/fusesoc/main.py @@ -5,6 +5,7 @@ import argparse import os +import pathlib import shutil import signal import sys @@ -303,6 +304,13 @@ def run(fs, args): else: flags[flag] = True + if args.lockfile is not None: + try: + fs.cm.db.load_lockfile(args.lockfile) + except SyntaxError as e: + logger.error(f"Failed to load lock file, {str(e)}") + exit(1) + core = _get_core(fs, args.system) try: @@ -615,6 +623,11 @@ def get_parser(): parser_run.add_argument( "backendargs", nargs=argparse.REMAINDER, help="arguments to be sent to backend" ) + parser_run.add_argument( + "--lockfile", + help="Lockfile file path", + type=pathlib.Path, + ) parser_run.set_defaults(func=run) # config subparser diff --git a/fusesoc/vlnv.py b/fusesoc/vlnv.py index 4925f7ba..255424cd 100644 --- a/fusesoc/vlnv.py +++ b/fusesoc/vlnv.py @@ -161,3 +161,47 @@ def __lt__(self, other): other.name, other.version, ) + + def vln_str(self): + """Returns a string with ::""" + return f"{self.vendor}:{self.library}:{self.name}" + + +def compare_relation(vlvn_a: Vlnv, relation: str, vlvn_b: Vlnv): + """Compare two VLVNs with the provided relation. Returns boolan.""" + from okonomiyaki.versions import EnpkgVersion + + valid_version = False + version_str = lambda v: f"{v.version}-{v.revision}" + if vlvn_a.vln_str() == vlvn_b.vln_str(): + ver_a = EnpkgVersion.from_string(version_str(vlvn_a)) + ver_b = EnpkgVersion.from_string(version_str(vlvn_b)) + if relation == "==": + valid_version = ver_a == ver_b + elif relation == ">": + valid_version = ver_a > ver_b + elif relation == "<": + valid_version = ver_a < ver_b + elif relation == ">=": + valid_version = ver_a >= ver_b + elif relation == "<=": + valid_version = ver_a <= ver_b + elif relation == "^": + nextversion = list(map(int, vlvn_a.version.split("."))) + for pos in range(len(nextversion)): + if pos == 0: + nextversion[pos] += 1 + else: + nextversion[pos] = 0 + nextversion = EnpkgVersion.from_string(".".join(map(str, nextversion))) + valid_version = ver_a <= ver_b and ver_b < nextversion + elif relation == "~": + nextversion = list(map(int, vlvn_a.version.split("."))) + for pos in range(len(nextversion)): + if pos == 1: + nextversion[pos] += 1 + elif pos > 1: + nextversion[pos] = 0 + nextversion = EnpkgVersion.from_string(".".join(map(str, nextversion))) + valid_version = ver_a <= ver_b and ver_b < nextversion + return valid_version diff --git a/tests/capi2_cores/dependencies/top.core b/tests/capi2_cores/dependencies/top.core new file mode 100644 index 00000000..8c514977 --- /dev/null +++ b/tests/capi2_cores/dependencies/top.core @@ -0,0 +1,18 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: ::dependencies-top + +filesets: + fs1: + depend: + - '>::used:1.0' + +targets: + default: + filesets: + - fs1 + toplevel: + - top diff --git a/tests/capi2_cores/dependencies/used-1.0.core b/tests/capi2_cores/dependencies/used-1.0.core new file mode 100644 index 00000000..aec34978 --- /dev/null +++ b/tests/capi2_cores/dependencies/used-1.0.core @@ -0,0 +1,18 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: ::used:1.0 +filesets: + rtl: + files: + - used-1.0.sv + file_type: systemVerilogSource + + +targets: + default: + filesets: + - rtl + toplevel: used_1_0 diff --git a/tests/capi2_cores/dependencies/used-1.1.core b/tests/capi2_cores/dependencies/used-1.1.core new file mode 100644 index 00000000..2e92b363 --- /dev/null +++ b/tests/capi2_cores/dependencies/used-1.1.core @@ -0,0 +1,18 @@ +CAPI=2: +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +name: ::used:1.1 +filesets: + rtl: + files: + - used-1.1.sv + file_type: systemVerilogSource + + +targets: + default: + filesets: + - rtl + toplevel: used_1_1 diff --git a/tests/lockfiles/dependencies-partial-1.0.lock.yml b/tests/lockfiles/dependencies-partial-1.0.lock.yml new file mode 100644 index 00000000..0a750459 --- /dev/null +++ b/tests/lockfiles/dependencies-partial-1.0.lock.yml @@ -0,0 +1,2 @@ +cores: + - name: "::used:1.0" diff --git a/tests/lockfiles/dependencies-partial.lock.yml b/tests/lockfiles/dependencies-partial.lock.yml new file mode 100644 index 00000000..18212221 --- /dev/null +++ b/tests/lockfiles/dependencies-partial.lock.yml @@ -0,0 +1,2 @@ +cores: + - name: "::used:1.1" diff --git a/tests/lockfiles/dependencies.lock.yml b/tests/lockfiles/dependencies.lock.yml new file mode 100644 index 00000000..ddc58e25 --- /dev/null +++ b/tests/lockfiles/dependencies.lock.yml @@ -0,0 +1,3 @@ +cores: + - name: "::used:1.1" + - name: "::dependencies-top:0" diff --git a/tests/lockfiles/duplicates.lock.yml b/tests/lockfiles/duplicates.lock.yml new file mode 100644 index 00000000..e570af48 --- /dev/null +++ b/tests/lockfiles/duplicates.lock.yml @@ -0,0 +1,8 @@ +lockfile_version: 1 +fusesoc_version: 2.4.2 +cores: +- name: ":lib:pin:0.1" +- name: ":lib:pin:0.2" +- name: ":lib:gpio:0.1" +- name: ":common:gpio_ctrl:0.1" +- name: ":product:toppy:0.1" diff --git a/tests/lockfiles/works.lock.yml b/tests/lockfiles/works.lock.yml new file mode 100644 index 00000000..e46be304 --- /dev/null +++ b/tests/lockfiles/works.lock.yml @@ -0,0 +1,7 @@ +lockfile_version: 1 +fusesoc_version: 2.4.2 +cores: +- name: ":lib:pin:0.1" +- name: ":lib:gpio:0.1" +- name: ":common:gpio_ctrl:0.1" +- name: ":product:toppy:0.1" diff --git a/tests/test_coremanager.py b/tests/test_coremanager.py index 679371fb..4019708a 100644 --- a/tests/test_coremanager.py +++ b/tests/test_coremanager.py @@ -365,3 +365,161 @@ def test_virtual_non_deterministic_virtual(caplog): "::user:0", "::top_non_deterministic:0", ] + + +def test_lockfile(caplog): + """ + Test core selection with a core pinned by a lock file + """ + import logging + import os + import pathlib + import tempfile + + from fusesoc.config import Config + from fusesoc.coremanager import CoreManager + from fusesoc.edalizer import Edalizer + from fusesoc.librarymanager import Library + from fusesoc.vlnv import Vlnv + + flags = {"tool": "icarus"} + + build_root = tempfile.mkdtemp(prefix="export_") + work_root = os.path.join(build_root, "work") + + core_dir = os.path.join(os.path.dirname(__file__), "capi2_cores", "dependencies") + + cm = CoreManager(Config()) + cm.add_library(Library("virtual", core_dir), []) + cm.db.load_lockfile( + pathlib.Path(__file__).parent / "lockfiles" / "dependencies.lock.yml" + ) + + root_core = cm.get_core(Vlnv("::dependencies-top")) + + edalizer = Edalizer( + toplevel=root_core.name, + flags=flags, + core_manager=cm, + work_root=work_root, + ) + + with caplog.at_level(logging.WARNING): + edalizer.run() + + assert caplog.records == [] + + deps = cm.get_depends(root_core.name, {}) + deps_names = [str(c) for c in deps] + + for dependency in deps_names: + assert dependency in [ + "::used:1.1", + "::dependencies-top:0", + ] + + +def test_lockfile_partial_warning(caplog): + """ + Test core selection with a core pinned by a lock file + """ + import logging + import os + import pathlib + import tempfile + + from fusesoc.config import Config + from fusesoc.coremanager import CoreManager + from fusesoc.edalizer import Edalizer + from fusesoc.librarymanager import Library + from fusesoc.vlnv import Vlnv + + flags = {"tool": "icarus"} + + build_root = tempfile.mkdtemp(prefix="export_") + work_root = os.path.join(build_root, "work") + + core_dir = os.path.join(os.path.dirname(__file__), "capi2_cores", "dependencies") + + cm = CoreManager(Config()) + cm.add_library(Library("virtual", core_dir), []) + cm.db.load_lockfile( + pathlib.Path(__file__).parent / "lockfiles" / "dependencies-partial.lock.yml" + ) + + root_core = cm.get_core(Vlnv("::dependencies-top")) + + edalizer = Edalizer( + toplevel=root_core.name, + flags=flags, + core_manager=cm, + work_root=work_root, + ) + + with caplog.at_level(logging.WARNING): + edalizer.run() + + assert "Using lock file with partial list of cores" in caplog.text + + deps = cm.get_depends(root_core.name, {}) + deps_names = [str(c) for c in deps] + + for dependency in deps_names: + assert dependency in [ + "::used:1.1", + "::dependencies-top:0", + ] + + +def test_lockfile_version_warning(caplog): + """ + Test core selection with a core pinned by a lock file, warning if version is out of scope + """ + import logging + import os + import pathlib + import tempfile + + from fusesoc.config import Config + from fusesoc.coremanager import CoreManager + from fusesoc.edalizer import Edalizer + from fusesoc.librarymanager import Library + from fusesoc.vlnv import Vlnv + + flags = {"tool": "icarus"} + + build_root = tempfile.mkdtemp(prefix="export_") + work_root = os.path.join(build_root, "work") + + core_dir = os.path.join(os.path.dirname(__file__), "capi2_cores", "dependencies") + + cm = CoreManager(Config()) + cm.add_library(Library("virtual", core_dir), []) + cm.db.load_lockfile( + pathlib.Path(__file__).parent + / "lockfiles" + / "dependencies-partial-1.0.lock.yml" + ) + + root_core = cm.get_core(Vlnv("::dependencies-top")) + + edalizer = Edalizer( + toplevel=root_core.name, + flags=flags, + core_manager=cm, + work_root=work_root, + ) + + with caplog.at_level(logging.WARNING): + edalizer.run() + + assert "Failed to pin" in caplog.text + + deps = cm.get_depends(root_core.name, {}) + deps_names = [str(c) for c in deps] + + for dependency in deps_names: + assert dependency in [ + "::used:1.1", + "::dependencies-top:0", + ] diff --git a/tests/test_lockfile.py b/tests/test_lockfile.py new file mode 100644 index 00000000..6fb03e86 --- /dev/null +++ b/tests/test_lockfile.py @@ -0,0 +1,34 @@ +# Copyright FuseSoC contributors +# Licensed under the 2-Clause BSD License, see LICENSE for details. +# SPDX-License-Identifier: BSD-2-Clause + +import os +import pathlib + +import pytest + +from fusesoc.lockfile import load_lockfile + + +def test_load_lockfile(): + from fusesoc.vlnv import Vlnv + + lockfile = load_lockfile( + pathlib.Path(__file__).parent / "lockfiles" / "works.lock.yml" + ) + + assert lockfile == { + "cores": { + Vlnv(":lib:pin:0.1"): {"name": Vlnv(":lib:pin:0.1")}, + Vlnv(":lib:gpio:0.1"): {"name": Vlnv(":lib:gpio:0.1")}, + Vlnv(":common:gpio_ctrl:0.1"): {"name": Vlnv(":common:gpio_ctrl:0.1")}, + Vlnv(":product:toppy:0.1"): {"name": Vlnv(":product:toppy:0.1")}, + }, + } + + +def test_load_lockfile_duplicates(): + with pytest.raises(SyntaxError): + _ = load_lockfile( + pathlib.Path(__file__).parent / "lockfiles" / "duplicates.lock.yml" + ) diff --git a/tests/test_vlnv.py b/tests/test_vlnv.py index 069ee0d5..76ecfcdb 100644 --- a/tests/test_vlnv.py +++ b/tests/test_vlnv.py @@ -4,7 +4,7 @@ import pytest -from fusesoc.vlnv import Vlnv +from fusesoc.vlnv import Vlnv, compare_relation def vlnv_tuple(vlnv): @@ -80,3 +80,51 @@ def test_name_version_revision_legacy(): def test_name_revision_legacy(): assert vlnv_tuple(Vlnv("uart16550-r2")) == ("", "", "uart16550", "0", 2) + + +def test_vlvn_compare_relation(): + version_1_3 = Vlnv(":peripherals:uart16550:1.3.1") + version_1_4 = Vlnv(":peripherals:uart16550:1.4.2") + version_1_5 = Vlnv(":peripherals:uart16550:1.5.1") + version_2_0 = Vlnv(":peripherals:uart16550:2.0.1") + + assert compare_relation(version_1_4, "==", version_1_4) + assert not compare_relation( + version_1_4, "==", Vlnv("other:peripherals:uart16550:1.4.2") + ) + assert not compare_relation(version_1_4, "==", Vlnv(":other:uart16550:1.4.2")) + assert not compare_relation(version_1_4, "==", Vlnv(":peripherals:other:1.4.2")) + assert not compare_relation(version_1_4, "==", version_1_3) + assert not compare_relation(version_1_4, "==", version_1_5) + assert not compare_relation(version_1_4, "==", version_2_0) + + assert compare_relation(version_1_4, ">", version_1_3) + assert not compare_relation(version_1_4, ">", version_1_4) + assert not compare_relation(version_1_4, ">", version_1_5) + assert not compare_relation(version_1_4, ">", version_2_0) + + assert compare_relation(version_1_4, ">=", version_1_3) + assert compare_relation(version_1_4, ">=", version_1_4) + assert not compare_relation(version_1_4, ">=", version_1_5) + assert not compare_relation(version_1_4, ">=", version_2_0) + + assert not compare_relation(version_1_4, "<", version_1_3) + assert not compare_relation(version_1_4, "<", version_1_4) + assert compare_relation(version_1_4, "<", version_1_5) + assert compare_relation(version_1_4, "<", version_2_0) + + assert not compare_relation(version_1_4, "<=", version_1_3) + assert compare_relation(version_1_4, "<=", version_1_4) + assert compare_relation(version_1_4, "<=", version_1_5) + assert compare_relation(version_1_4, "<=", version_2_0) + + assert not compare_relation(version_1_4, "^", version_1_3) + assert compare_relation(version_1_4, "^", version_1_4) + assert compare_relation(version_1_4, "^", version_1_5) + assert not compare_relation(version_1_4, "^", version_2_0) + + assert not compare_relation(version_1_4, "~", version_1_3) + assert compare_relation(version_1_4, "~", version_1_4) + assert compare_relation(version_1_4, "~", Vlnv(":peripherals:uart16550:1.4.9")) + assert not compare_relation(version_1_4, "~", version_1_5) + assert not compare_relation(version_1_4, "~", version_2_0)