Merge pull request #728 from blueluna/lockfile

Adding Lockfile

Author: olofk

fusesoc/coremanager.py

@@ -4,6 +4,7 @@
 
 import logging
 import os
+import pathlib
 
 from okonomiyaki.versions import EnpkgVersion
 from simplesat.constraints import PrettyPackageStringParser, Requirement
@@ -16,6 +17,8 @@
 from fusesoc.capi2.coreparser import Core2Parser
 from fusesoc.core import Core
 from fusesoc.librarymanager import LibraryManager
+from fusesoc.lockfile import load_lockfile
+from fusesoc.vlnv import Vlnv, compare_relation
 
 logger = logging.getLogger(__name__)
 
@@ -33,6 +36,7 @@ class CoreDB:
     def __init__(self):
         self._cores = {}
         self._solver_cache = {}
+        self._lockfile = None
 
     # simplesat doesn't allow ':', '-' or leading '_'
     def _package_name(self, vlnv):
@@ -45,6 +49,7 @@ def _package_version(self, vlnv):
     def _parse_depend(self, depends):
         # FIXME: Handle conflicts
         deps = []
+
         _s = "{} {} {}"
         for d in depends:
             for simple in d.simpleVLNVs():
@@ -83,6 +88,9 @@ def find(self, vlnv=None):
             found = list([core["core"] for core in self._cores.values()])
         return found
 
+    def load_lockfile(self, filepath: pathlib.Path):
+        self._lockfile = load_lockfile(filepath)
+
     def _solver_cache_lookup(self, key):
         if key in self._solver_cache:
             return self._solver_cache[key]
@@ -110,6 +118,27 @@ def _hash_flags_dict(self, flags):
             h ^= hash(pair)
         return h
 
+    def _lockfile_replace(self, core: Vlnv):
+        """Try to pin the core version from cores defined in the lock file"""
+        if self._lockfile:
+            for locked_core in self._lockfile["cores"]:
+                if locked_core.vln_str() == core.vln_str():
+                    valid_version = compare_relation(locked_core, core.relation, core)
+                    if valid_version:
+                        core.version = locked_core.version
+                        core.revision = locked_core.revision
+                        core.relation = "=="
+                    else:
+                        # Invalid version in lockfile
+                        logger.warning(
+                            "Failed to pin core {} outside of dependency version {} {} {}".format(
+                                str(locked_core),
+                                core.vln_str(),
+                                core.relation,
+                                core.version,
+                            )
+                        )
+
     def solve(self, top_core, flags):
         return self._solve(top_core, flags)
 
@@ -195,8 +224,12 @@ def eq_vln(this, that):
                 _flags["is_toplevel"] = core.name == top_core
                 _depends = core.get_depends(_flags)
                 if _depends:
+                    for depend in _depends:
+                        self._lockfile_replace(depend)
                     _s = "; depends ( {} )"
                     package_str += _s.format(self._parse_depend(_depends))
+            else:
+                self._lockfile_replace(top_core)
 
             parser = PrettyPackageStringParser(EnpkgVersion.from_string)
 
@@ -226,6 +259,7 @@ def eq_vln(this, that):
             raise DependencyError(top_core.name)
 
         virtual_selection = {}
+        partial_lockfile = False
         objdict = {}
         if len(transaction.operations) > 1:
             for op in transaction.operations:
@@ -244,6 +278,11 @@ def eq_vln(this, that):
                     if p[0] in virtual_selection:
                         # If package that implements a virtual core is required, remove from the dictionary
                         del virtual_selection[p[0]]
+                if (
+                    self._lockfile
+                    and op.package.core.name not in self._lockfile["cores"]
+                ):
+                    partial_lockfile = True
                 op.package.core.direct_deps = [
                     objdict[n[0]] for n in op.package.install_requires
                 ]
@@ -254,6 +293,8 @@ def eq_vln(this, that):
                     virtual[1], virtual[0]
                 )
             )
+        if partial_lockfile:
+            logger.warning("Using lock file with partial list of cores")
 
         result = [op.package.core for op in transaction.operations]
 

fusesoc/lockfile.py

@@ -0,0 +1,76 @@
+import json
+import logging
+import os
+import pathlib
+
+import fastjsonschema
+
+import fusesoc.utils
+from fusesoc.version import version
+from fusesoc.vlnv import Vlnv
+
+logger = logging.getLogger(__name__)
+
+lockfile_schema = """
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "FuseSoC Lockfile",
+    "description": "FuseSoC Lockfile",
+    "type": "object",
+    "properties": {
+        "cores": {
+            "description": "Cores used in the build",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "description": "Core VLVN",
+                    "type": "string"
+                  }
+                }
+            }
+        },
+        "fusesoc_version": {
+            "description": "FuseSoC version which generated the lockfile",
+            "type": "string"
+        },
+        "lockfile_version": {
+            "description": "Lockfile version",
+            "type": "integer"
+        }
+    }
+}
+"""
+
+
+def load_lockfile(filepath: pathlib.Path):
+    try:
+        lockfile_data = fusesoc.utils.yaml_fread(filepath)
+        try:
+            validator = fastjsonschema.compile(
+                json.loads(lockfile_schema), detailed_exceptions=False
+            )
+            validator(lockfile_data)
+        except fastjsonschema.JsonSchemaDefinitionException as e:
+            raise SyntaxError(f"Error parsing JSON Schema: {e}")
+        except fastjsonschema.JsonSchemaException as e:
+            raise SyntaxError(f"Error validating {e}")
+    except FileNotFoundError:
+        logger.warning(f"Lockfile {filepath} not found")
+        return None
+
+    cores = {}
+    for core in lockfile_data.setdefault("cores", []):
+        if "name" in core:
+            vlnv = Vlnv(core["name"])
+            vln = vlnv.vln_str()
+            if vln in map(Vlnv.vln_str, cores.keys()):
+                raise SyntaxError(f"Core {vln} defined multiple times in lock file")
+            cores[vlnv] = {"name": vlnv}
+        else:
+            raise SyntaxError(f"Core definition without a name")
+    lockfile = {
+        "cores": cores,
+    }
+    return lockfile

fusesoc/main.py

@@ -6,6 +6,7 @@
 
 import argparse
 import os
+import pathlib
 import shutil
 import signal
 import sys
@@ -306,6 +307,13 @@ def run(fs, args):
         else:
             flags[flag] = True
 
+    if args.lockfile is not None:
+        try:
+            fs.cm.db.load_lockfile(args.lockfile)
+        except SyntaxError as e:
+            logger.error(f"Failed to load lock file, {str(e)}")
+            exit(1)
+
     core = _get_core(fs, args.system)
 
     try:
@@ -666,6 +674,11 @@ def get_parser():
     parser_run.add_argument(
         "backendargs", nargs=argparse.REMAINDER, help="arguments to be sent to backend"
     )
+    parser_run.add_argument(
+        "--lockfile",
+        help="Lockfile file path",
+        type=pathlib.Path,
+    )
     parser_run.set_defaults(func=run)
 
     # config subparser

fusesoc/vlnv.py

@@ -161,3 +161,47 @@ def __lt__(self, other):
             other.name,
             other.version,
         )
+
+    def vln_str(self):
+        """Returns a string with <vendor>:<library>:<name>"""
+        return f"{self.vendor}:{self.library}:{self.name}"
+
+
+def compare_relation(vlvn_a: Vlnv, relation: str, vlvn_b: Vlnv):
+    """Compare two VLVNs with the provided relation. Returns boolan."""
+    from okonomiyaki.versions import EnpkgVersion
+
+    valid_version = False
+    version_str = lambda v: f"{v.version}-{v.revision}"
+    if vlvn_a.vln_str() == vlvn_b.vln_str():
+        ver_a = EnpkgVersion.from_string(version_str(vlvn_a))
+        ver_b = EnpkgVersion.from_string(version_str(vlvn_b))
+        if relation == "==":
+            valid_version = ver_a == ver_b
+        elif relation == ">":
+            valid_version = ver_a > ver_b
+        elif relation == "<":
+            valid_version = ver_a < ver_b
+        elif relation == ">=":
+            valid_version = ver_a >= ver_b
+        elif relation == "<=":
+            valid_version = ver_a <= ver_b
+        elif relation == "^":
+            nextversion = list(map(int, vlvn_a.version.split(".")))
+            for pos in range(len(nextversion)):
+                if pos == 0:
+                    nextversion[pos] += 1
+                else:
+                    nextversion[pos] = 0
+            nextversion = EnpkgVersion.from_string(".".join(map(str, nextversion)))
+            valid_version = ver_a <= ver_b and ver_b < nextversion
+        elif relation == "~":
+            nextversion = list(map(int, vlvn_a.version.split(".")))
+            for pos in range(len(nextversion)):
+                if pos == 1:
+                    nextversion[pos] += 1
+                elif pos > 1:
+                    nextversion[pos] = 0
+            nextversion = EnpkgVersion.from_string(".".join(map(str, nextversion)))
+            valid_version = ver_a <= ver_b and ver_b < nextversion
+    return valid_version

tests/capi2_cores/dependencies/top.core

@@ -0,0 +1,18 @@
+CAPI=2:
+# Copyright FuseSoC contributors
+# Licensed under the 2-Clause BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-2-Clause
+
+name: ::dependencies-top
+
+filesets:
+  fs1:
+    depend:
+      - '>::used:1.0'
+
+targets:
+  default:
+    filesets:
+      - fs1
+    toplevel:
+      - top

tests/capi2_cores/dependencies/used-1.0.core

@@ -0,0 +1,18 @@
+CAPI=2:
+# Copyright FuseSoC contributors
+# Licensed under the 2-Clause BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-2-Clause
+
+name: ::used:1.0
+filesets:
+  rtl:
+    files:
+      - used-1.0.sv
+    file_type: systemVerilogSource
+
+
+targets:
+  default:
+    filesets:
+      - rtl
+    toplevel: used_1_0

tests/capi2_cores/dependencies/used-1.1.core

@@ -0,0 +1,18 @@
+CAPI=2:
+# Copyright FuseSoC contributors
+# Licensed under the 2-Clause BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-2-Clause
+
+name: ::used:1.1
+filesets:
+  rtl:
+    files:
+      - used-1.1.sv
+    file_type: systemVerilogSource
+
+
+targets:
+  default:
+    filesets:
+      - rtl
+    toplevel: used_1_1

tests/lockfiles/dependencies-partial-1.0.lock.yml

@@ -0,0 +1,2 @@
+cores:
+  - name: "::used:1.0"

tests/lockfiles/dependencies-partial.lock.yml

@@ -0,0 +1,2 @@
+cores:
+  - name: "::used:1.1"

tests/lockfiles/dependencies.lock.yml

@@ -0,0 +1,3 @@
+cores:
+  - name: "::used:1.1"
+  - name: "::dependencies-top:0"