Adding Lockfile

fusesoc/coremanager.py

@@ -4,6 +4,7 @@
 
 import logging
 import os
+import pathlib
 
 from okonomiyaki.versions import EnpkgVersion
 from simplesat.constraints import PrettyPackageStringParser, Requirement
@@ -13,9 +14,12 @@
 from simplesat.repository import Repository
 from simplesat.request import Request
 
+import fusesoc.lockfile
 from fusesoc.capi2.coreparser import Core2Parser
 from fusesoc.core import Core
 from fusesoc.librarymanager import LibraryManager
+from fusesoc.lockfile import load_lockfile
+from fusesoc.vlnv import compare_relation
 
 logger = logging.getLogger(__name__)
 
@@ -33,6 +37,7 @@ class CoreDB:
     def __init__(self):
         self._cores = {}
         self._solver_cache = {}
+        self._lockfile = None
 
     # simplesat doesn't allow ':', '-' or leading '_'
     def _package_name(self, vlnv):
@@ -45,6 +50,7 @@ def _package_version(self, vlnv):
     def _parse_depend(self, depends):
         # FIXME: Handle conflicts
         deps = []
+
         _s = "{} {} {}"
         for d in depends:
             for simple in d.simpleVLNVs():
@@ -83,6 +89,9 @@ def find(self, vlnv=None):
             found = list([core["core"] for core in self._cores.values()])
         return found
 
+    def load_lockfile(self, filepath: pathlib.Path):
+        self._lockfile = load_lockfile(filepath)
+
     def _solver_cache_lookup(self, key):
         if key in self._solver_cache:
             return self._solver_cache[key]
@@ -161,6 +170,15 @@ def eq_vln(this, that):
         cores = [x["core"] for x in self._cores.values()]
         conflict_map = self._get_conflict_map()
 
+        lockfile_virtuals = {}
+
+        if isinstance(self._lockfile, dict):
+            for pin_core in self._lockfile["cores"]:
+                if str(pin_core) in self._cores:
+                    core = self._cores[str(pin_core)]["core"]
+                    for virtual in core.get_virtuals():
+                        lockfile_virtuals[virtual] = core.name
+
         for core in cores:
             if only_matching_vlnv:
                 if not any(
@@ -195,6 +213,38 @@ def eq_vln(this, that):
                 _flags["is_toplevel"] = core.name == top_core
                 _depends = core.get_depends(_flags)
                 if _depends:
+                    for depend in _depends:
+                        virtual_selection = None
+                        if isinstance(self._lockfile, dict):
+                            if depend in self._lockfile["virtuals"]:
+                                implementation_core = self._lockfile["virtuals"][depend]
+                                virtual_selection = implementation_core
+                            elif depend in lockfile_virtuals:
+                                implementation_core = lockfile_virtuals[depend]
+                                virtual_selection = implementation_core
+                            else:
+                                for locked_core in self._lockfile["cores"]:
+                                    if locked_core.vln_str() == depend.vln_str():
+                                        valid_version = compare_relation(
+                                            locked_core, depend.relation, depend
+                                        )
+                                        if valid_version:
+                                            depend.version = locked_core.version
+                                            depend.revision = locked_core.revision
+                                            depend.relation = "=="
+                                        else:
+                                            # Invalid version in lockfile
+                                            logger.warning(
+                                                "Failed to pin core {} outside of dependency version {} {} {}".format(
+                                                    str(locked_core),
+                                                    depend.vln_str(),
+                                                    depend.relation,
+                                                    depend.version,
+                                                )
+                                            )
+                        if virtual_selection:
+                            _depends.append(virtual_selection)
+                            _depends.remove(depend)
                     _s = "; depends ( {} )"
                     package_str += _s.format(self._parse_depend(_depends))
 

fusesoc/lockfile.py

@@ -0,0 +1,78 @@
+import json
+import logging
+import os
+import pathlib
+
+import fastjsonschema
+
+import fusesoc.utils
+from fusesoc.version import version
+from fusesoc.vlnv import Vlnv
+
+logger = logging.getLogger(__name__)
+
+lockfile_schema = """
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "FuseSoC Lockfile",
+    "description": "FuseSoC Lockfile",
+    "type": "object",
+    "properties": {
+        "cores": {
+            "description": "Cores used in the build",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+        },
+        "virtuals": {
+            "description": "Virtual cores used in the build",
+            "type": "object",
+            "patternProperties": {
+                "^.+$": {
+                    "description": "Virtual core used in the build",
+                    "patternProperties": {
+                        "^.+$": {
+                            "description": "Core implementing virtual core interface"
+                        }
+                    }
+                }
+            }
+        },
+        "fusesoc_version": {
+            "description": "FuseSoC version which generated the lockfile",
+            "type": "string"
+        },
+        "lockfile_version": {
+            "description": "Lockfile version",
+            "type": "integer"
+        }
+    }
+}
+"""
+
+
+def load_lockfile(filepath: pathlib.Path):
+    try:
+        lockfile_data = fusesoc.utils.yaml_fread(filepath)
+        try:
+            validator = fastjsonschema.compile(json.loads(lockfile_schema))
+            validator(lockfile_data)
+        except fastjsonschema.JsonSchemaDefinitionException as e:
+            raise SyntaxError(f"\nError parsing JSON Schema: {e}")
+        except fastjsonschema.JsonSchemaException as e:
+            raise SyntaxError(f"\nError validating {e}")
+    except FileNotFoundError:
+        return None
+
+    cores = [Vlnv(core_name) for core_name in lockfile_data.setdefault("cores", [])]
+    virtuals = {
+        Vlnv(virtual_name): Vlnv(core_name)
+        for virtual_name, core_name in lockfile_data.setdefault("virtuals", {}).items()
+    }
+    lockfile = {
+        "cores": cores,
+        "virtuals": virtuals,
+    }
+
+    return lockfile

fusesoc/main.py

@@ -5,6 +5,7 @@
 
 import argparse
 import os
+import pathlib
 import shutil
 import signal
 import sys
@@ -303,6 +304,9 @@ def run(fs, args):
         else:
             flags[flag] = True
 
+    if args.lockfile is not None:
+        fs.cm.db.load_lockfile(args.lockfile)
+
     core = _get_core(fs, args.system)
 
     try:
@@ -615,6 +619,11 @@ def get_parser():
     parser_run.add_argument(
         "backendargs", nargs=argparse.REMAINDER, help="arguments to be sent to backend"
     )
+    parser_run.add_argument(
+        "--lockfile",
+        help="Lockfile file path",
+        type=pathlib.Path,
+    )
     parser_run.set_defaults(func=run)
 
     # config subparser

fusesoc/vlnv.py

@@ -161,3 +161,53 @@ def __lt__(self, other):
             other.name,
             other.version,
         )
+
+    def vln_str(self):
+        """Returns a string with <vendor>:<library>:<name>"""
+        return f"{self.vendor}:{self.library}:{self.name}"
+
+
+def compare_relation(vlvn_a, relation, vlvn_b):
+    """Compare two VLVNs with the provided relation. Returns boolan."""
+    from okonomiyaki.versions import EnpkgVersion
+
+    if (
+        not isinstance(vlvn_a, Vlnv)
+        or not isinstance(relation, str)
+        or not isinstance(vlvn_b, Vlnv)
+    ):
+        return False
+    valid_version = False
+    version_str = lambda v: f"{v.version}-{v.revision}"
+    if vlvn_a.vln_str() == vlvn_b.vln_str():
+        ver_a = EnpkgVersion.from_string(version_str(vlvn_a))
+        ver_b = EnpkgVersion.from_string(version_str(vlvn_b))
+        if relation == "==":
+            valid_version = ver_a == ver_b
+        elif relation == ">":
+            valid_version = ver_a > ver_b
+        elif relation == "<":
+            valid_version = ver_a < ver_b
+        elif relation == ">=":
+            valid_version = ver_a >= ver_b
+        elif relation == "<=":
+            valid_version = ver_a <= ver_b
+        elif relation == "^":
+            nextversion = list(map(int, vlvn_a.version.split(".")))
+            for pos in range(len(nextversion)):
+                if pos == 0:
+                    nextversion[pos] += 1
+                else:
+                    nextversion[pos] = 0
+            nextversion = EnpkgVersion.from_string(".".join(map(str, nextversion)))
+            valid_version = ver_a <= ver_b and ver_b < nextversion
+        elif relation == "~":
+            nextversion = list(map(int, vlvn_a.version.split(".")))
+            for pos in range(len(nextversion)):
+                if pos == 1:
+                    nextversion[pos] += 1
+                elif pos > 1:
+                    nextversion[pos] = 0
+            nextversion = EnpkgVersion.from_string(".".join(map(str, nextversion)))
+            valid_version = ver_a <= ver_b and ver_b < nextversion
+    return valid_version

tests/capi2_cores/dependencies/top.core

@@ -0,0 +1,18 @@
+CAPI=2:
+# Copyright FuseSoC contributors
+# Licensed under the 2-Clause BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-2-Clause
+
+name: ::dependencies-top
+
+filesets:
+  fs1:
+    depend:
+      - '>::used:1.0'
+
+targets:
+  default:
+    filesets:
+      - fs1
+    toplevel:
+      - top

tests/capi2_cores/dependencies/used-1.0.core

@@ -0,0 +1,18 @@
+CAPI=2:
+# Copyright FuseSoC contributors
+# Licensed under the 2-Clause BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-2-Clause
+
+name: ::used:1.0
+filesets:
+  rtl:
+    files:
+      - used-1.0.sv
+    file_type: systemVerilogSource
+
+
+targets:
+  default:
+    filesets:
+      - rtl
+    toplevel: used_1_0

tests/capi2_cores/dependencies/used-1.1.core

@@ -0,0 +1,18 @@
+CAPI=2:
+# Copyright FuseSoC contributors
+# Licensed under the 2-Clause BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-2-Clause
+
+name: ::used:1.1
+filesets:
+  rtl:
+    files:
+      - used-1.1.sv
+    file_type: systemVerilogSource
+
+
+targets:
+  default:
+    filesets:
+      - rtl
+    toplevel: used_1_1

tests/lockfiles/dependencies-1.0.lock

@@ -0,0 +1,2 @@
+cores:
+  - ::used:1.0

tests/lockfiles/dependencies.lock

@@ -0,0 +1,2 @@
+cores:
+  - ::used:1.1

tests/lockfiles/virtual.lock

@@ -0,0 +1,2 @@
+virtuals:
+  ::someinterface:0: ::impl2:0

tests/lockfiles/works.lock

@@ -0,0 +1,10 @@
+lockfile_version: 1
+fusesoc_version: 2.4.2
+cores:
+- :lib:pin:0.1
+- :lib:gpio:0.1
+- :common:gpio_ctrl:0.1
+- :product:toppy:0.1
+virtuals:
+  :interface:pin: :lib:pin:0.1
+  :interface:gpio: :lib:gpio:0.1