Add diff_definitions.py for diffing event definition files and schema…

… files (#389)

This new script makes it easy to compare the current definitions
of events and other types against a chosen base, e.g. the current
state of the master branch or a previous edition. It produces diff
commands that can be run to produce the wanted comparison.

Since figuring out the most recent version of each type is a pretty
common operation, a new versions module is introduced for this purpose.
The existing find-latest-schemas.py script has been adapted to use the
new module.

diff_definitions.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+# Copyright 2024 Axis Communications AB.
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Context-sensitive diffing of Eiffel type definitions. Compares the
+current workspace's latest type versions with the ones from a specified
+base, and prints diff commands. For example, if the current commit has
+added v4.3.0 of ActT and the given base had v4.2.0 as its latest version,
+you'll get the following output:
+
+diff -u definitions/EiffelActivityTriggeredEvent/4.2.0.yml definitions/EiffelActivityTriggeredEvent/4.3.0.yml
+diff -u schemas/EiffelActivityTriggeredEvent/4.2.0.json schemas/EiffelActivityTriggeredEvent/4.3.0.json
+
+By default, the base of the comparison is origin/master, but any commit
+reference can be given as an argument.
+"""
+
+import sys
+from pathlib import Path
+
+import versions
+
+
+def _main():
+    base = "origin/master"
+    if len(sys.argv) > 2:
+        print(f"Usage: python {sys.argv[0]} [ base ]")
+        sys.exit(-1)
+    elif len(sys.argv) == 2:
+        base = sys.argv[1]
+
+    base_defs = versions.latest_in_gitref(base, ".", Path("definitions"))
+    workspace_defs = versions.latest_in_dir(Path("definitions"))
+    for type, workspace_version in sorted(workspace_defs.items()):
+        base_version = base_defs.get(type)
+        if not base_version:
+            print(f"diff -u /dev/null definitions/{type}/{workspace_version}.yml")
+            print(f"diff -u /dev/null schemas/{type}/{workspace_version}.json")
+        elif base_version != workspace_version:
+            print(
+                f"diff -u definitions/{type}/{base_version}.yml definitions/{type}/{workspace_version}.yml"
+            )
+            print(
+                f"diff -u schemas/{type}/{base_version}.json schemas/{type}/{workspace_version}.json"
+            )
+
+
+if __name__ == "__main__":
+    _main()

find-latest-schemas.py

@@ -1,8 +1,23 @@
-import os
+# Copyright 2016-2024 Ericsson AB and others.
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import sys
+from pathlib import Path
 from shutil import copyfile
 
-import semver
+import versions
 
 """
 Finds the latest versions of schemas found under <input_folder>, with
@@ -16,28 +31,19 @@
 
 def main():
     if len(sys.argv) != 3:
-        print("Usage: python {} input_folder output_folder".format(sys.argv[0]))
+        print(f"Usage: python {sys.argv[0]} input_folder output_folder")
         sys.exit(-1)
 
-    input_folder = sys.argv[1]
-    output_folder = sys.argv[2]
-
-    if not os.path.exists(output_folder):
-        os.makedirs(output_folder)
-
-    latest_versions = []
-    for base, _, files in os.walk(input_folder):
-        if len(files) != 0:
-            latest_version = "0.0.0"
-            for f in files:
-                latest_version = semver.max_ver(latest_version, os.path.splitext(f)[0])
-            latest_versions.append(os.path.join(base, latest_version + ".json"))
-
-    for f in latest_versions:
-        new_name = os.path.split(os.path.dirname(f))[1] + ".json"
-        output_file = os.path.join(output_folder, new_name)
-        copyfile(f, output_file)
-        print("{} => {}".format(f, output_file))
+    input_folder = Path(sys.argv[1])
+    output_folder = Path(sys.argv[2])
+
+    output_folder.mkdir(exist_ok=True)
+
+    for type, version in versions.latest_in_dir(input_folder).items():
+        input_file = input_folder / type / f"{version}.json"
+        output_file = output_folder / f"{type}.json"
+        copyfile(input_file, output_file)
+        print(f"{input_file} => {output_file}")
 
 
 if __name__ == "__main__":

test_versions.py

@@ -0,0 +1,95 @@
+# Copyright 2024 Axis Communications AB.
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+from pathlib import Path
+
+import pytest
+import semver
+
+import versions
+
+
+class Git:
+    """Simple class for running Git commands in a given directory."""
+
+    def __init__(self, path: Path):
+        self.path = path
+        self.command("init")
+        # "git commit" requires that we identify ourselves.
+        self.command("config", "user.name", "John Doe")
+        self.command("config", "user.email", "[email protected]")
+
+    def command(self, *args: str) -> None:
+        subprocess.check_call(["git"] + list(args), cwd=self.path)
+
+
+@pytest.fixture
+def tmp_git(tmp_path):
+    """Injects a Git instance rooted in a temporary directory."""
+    yield Git(tmp_path)
+
+
+def create_files(base_path: Path, *args: str) -> None:
+    for p in args:
+        fullpath = base_path / p
+        fullpath.parent.mkdir(parents=True, exist_ok=True)
+        fullpath.touch()
+
+
+def test_latest_in_gitref(tmp_git):
+    # Create a bunch of files in the git, commit them, and tag that commit.
+    create_files(
+        tmp_git.path,
+        "subdir_c/6.0.0.json",
+        "definitions/subdir_a/1.0.0.yml",
+        "definitions/subdir_a/2.0.0.yml",
+        "definitions/subdir_a/3.0.0.othersuffix",
+        "definitions/subdir_b/3.0.0.yml",
+        "definitions/subdir_b/4.0.0.yml",
+    )
+    tmp_git.command("add", "-A")
+    tmp_git.command("commit", "-m", "Initial set of files")
+    tmp_git.command("tag", "v1.0.0")
+
+    # Add an additional file and delete one of the original files.
+    (tmp_git.path / "definitions/subdir_b/5.0.0.yml").touch()
+    tmp_git.command("rm", "definitions/subdir_a/2.0.0.yml")
+    tmp_git.command("add", "-A")
+    tmp_git.command("commit", "-m", "Make changes")
+
+    # Make sure the results we get are consistent with the original
+    # contents of the git.
+    assert versions.latest_in_gitref("v1.0.0", tmp_git.path, "definitions") == {
+        "subdir_a": semver.VersionInfo.parse("2.0.0"),
+        "subdir_b": semver.VersionInfo.parse("4.0.0"),
+    }
+
+
+def test_latest_in_dir(tmp_path):
+    create_files(
+        tmp_path,
+        "subdir_c/6.0.0.yml",
+        "definitions/subdir_a/1.0.0.yml",
+        "definitions/subdir_a/2.0.0.yml",
+        "definitions/subdir_a/3.0.0.othersuffix",
+        "definitions/subdir_b/3.0.0.yml",
+        "definitions/subdir_b/4.0.0.yml",
+    )
+
+    assert versions.latest_in_dir(tmp_path / "definitions") == {
+        "subdir_a": semver.VersionInfo.parse("2.0.0"),
+        "subdir_b": semver.VersionInfo.parse("4.0.0"),
+    }

versions.py

@@ -0,0 +1,72 @@
+# Copyright 2024 Axis Communications AB.
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The versions module contains functions for discovering definition files."""
+
+import os
+import subprocess
+from pathlib import Path
+from typing import Dict
+from typing import Iterable
+
+import semver
+
+
+def latest_in_gitref(
+    committish: str, gitdir: Path, subdir: Path
+) -> Dict[str, semver.version.Version]:
+    """Lists the definition files found under a given subdirectory of a
+    git at a given point in time (described by a committish, e.g. a
+    SHA-1, tag, or branch reference) and returns a dict that maps each
+    typename (e.g. EiffelArtifactCreatedEvent) to the latest version found.
+    """
+    return _latest_versions(
+        Path(line)
+        for line in (
+            subprocess.check_output(
+                ["git", "ls-tree", "-r", "--name-only", committish, "--", subdir],
+                cwd=gitdir,
+            )
+            .decode("utf-8")
+            .splitlines()
+        )
+        if Path(line).suffix == ".yml"
+    )
+
+
+def latest_in_dir(path: Path) -> Dict[str, semver.version.Version]:
+    """Inspects the definition files found under a given path and returns
+    a dict that maps each typename (e.g. EiffelArtifactCreatedEvent) to
+    its latest version found.
+    """
+    return _latest_versions(
+        Path(current) / f
+        for current, _, files in os.walk(path)
+        for f in files
+        if Path(f).suffix == ".yml"
+    )
+
+
+def _latest_versions(paths: Iterable[Path]) -> Dict[str, semver.version.Version]:
+    """Given a list of foo/<typename>/<version>.<ext> pathnames, returns
+    a dict mapping typenames to the most recent version of that type.
+    """
+    result = {}
+    for path in paths:
+        type = path.parent.name
+        this_version = semver.VersionInfo.parse(Path(path.name).stem)
+        if type not in result or result[type] < this_version:
+            result[type] = this_version
+    return result