feat(scripts): Add compare and create_patches scripts + script refactors and cleanups

.github/CODEOWNERS

@@ -126,13 +126,18 @@ vercel.json @iotaledger/tooling
 /.github/CODEOWNERS @luca-moser @lzpap @miker83z @alexsporn
 
 # Scripts
-/scripts/dependency_graphs/ @muXxer
 /scripts/cargo_sort/ @muXxer
-/scripts/generate_files/ @muXxer
 /scripts/codesearch/ @muXxer
+/scripts/compare/ @muXxer
+/scripts/create_patches/ @muXxer
+/scripts/dependency_graphs/ @muXxer
+/scripts/generate_files/ @muXxer
 /scripts/slipstream/ @muXxer
 /scripts/tooling/ @iotaledger/tooling
+/scripts/track_upstream_commits/ @muXxer
+/scripts/utils/ @muXxer
 
+# Other
 /kiosk/ @iotaledger/vm-language
 /nre/ @iotaledger/node
 

.gitignore

@@ -37,6 +37,7 @@ Move.lock
 
 # Python
 .pyc
+**/__pycache__/
 
 # Node.js
 node_modules/
@@ -71,7 +72,6 @@ yarn-error.log*
 .env
 checkpoints_dir/*
 light_client.yaml
-scripts/dependency_graphs/output/*
 
 lcov.info
 **/venv/

docker/utils/build-script.sh

@@ -3,19 +3,16 @@
 # Modifications Copyright (c) 2024 IOTA Stiftung
 # SPDX-License-Identifier: Apache-2.0
 
-# Get the directory where build-script.sh is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# Source common.sh from the utils directory
-source "$SCRIPT_DIR/common.sh"
-
 # fast fail.
 set -e
 
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+
+# Source common.sh from the utils directory
+source "$REPO_ROOT/scripts/utils/common.sh"
+
 # Get the current working directory where the script was called
 CURRENT_WORKING_DIR="$(pwd)"
-
-REPO_ROOT="$(git rev-parse --show-toplevel)"
 DOCKERFILE="$CURRENT_WORKING_DIR/Dockerfile"
 GIT_REVISION="$(git describe --always --abbrev=12 --dirty --exclude '*')"
 BUILD_DATE="$(date -u +'%Y-%m-%d')"

scripts/cargo_sort/cargo_sort.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2024 IOTA Stiftung
+# SPDX-License-Identifier: Apache-2.0
 import os, re, argparse, subprocess
 
 COMMENT_DEPENDENCIES_START_EXTERNAL = "# external dependencies"

scripts/codesearch/codesearch.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2024 IOTA Stiftung
+# SPDX-License-Identifier: Apache-2.0
 import os, re
 from tqdm import tqdm
 from concurrent.futures import ProcessPoolExecutor, as_completed

scripts/codesearch/run_search_tests.sh

@@ -5,6 +5,6 @@
 #    --output                       # Output file to save the results.
 #    --verbose                      # Display detailed file path and line number for each occurrence.
 #    --debug                        # Display the line where the occurrence was found.
-source python_venv_wrapper.sh
+source ../utils/python_venv_wrapper.sh
 
 $PYTHON_CMD search_tests.py --regex "zklogin" --verbose --debug

scripts/codesearch/run_search_versions.sh

@@ -4,6 +4,6 @@
 #    --output                       # Output file to save the results.
 #    --verbose                      # Display detailed file path and line number for each occurrence.
 #    --debug                        # Display the line where the occurrence was found.
-source python_venv_wrapper.sh
+source ../utils/python_venv_wrapper.sh
 
 $PYTHON_CMD search_versions.py --verbose #--debug

scripts/codesearch/search_tests.py

@@ -1,4 +1,6 @@
-import re, argparse, time
+# Copyright (c) 2024 IOTA Stiftung
+# SPDX-License-Identifier: Apache-2.0
+import re, argparse
 from collections import defaultdict
 
 # import all funcs from codesearch.py
@@ -108,10 +110,19 @@ def merge_results_func(results, result):
     results = search_files_in_parallel(
         target_dir=args.target,
         ignored_dirs=[
+            '.cargo',
+            '.changeset',
+            '.config',
             '.git',
+            '.github',
+            '.husky',
+            '.pnpm_store',
+            '.vscode',
+            'chocolatey',
+            'linting',
             'scripts',
+            'target',
             'node_modules',
-            '.pnpm_store',
         ],
         file_pattern=re.compile(r'\.rs$', flags=re.IGNORECASE),
         search_in_file_func=search_in_file_func,

scripts/codesearch/search_versions.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2024 IOTA Stiftung
+# SPDX-License-Identifier: Apache-2.0
 import os, re, argparse
 from collections import defaultdict
 
@@ -76,11 +78,16 @@ def merge_results_func(results, result):
     results = search_files_in_parallel(
         target_dir=args.target,
         ignored_dirs=[
+            '.cargo',
+            '.changeset',
+            '.config',
             '.git',
-            'scripts',
-            'node_modules',
+            '.github',
+            '.husky',
             '.pnpm_store',
-            'unit_tests',
+            '.vscode',
+            'chocolatey',
+            'linting',
             'move-compiler',
             'move-vm-types',
             'move-bytecode-verifier',
@@ -90,6 +97,10 @@ def merge_results_func(results, result):
             'move-model',
             'move-prover',
             'move-vm-integration-tests',
+            'node_modules',
+            'scripts',
+            'target',
+            'unit_tests',
         ],
         file_pattern=re.compile(r'\.rs$', flags=re.IGNORECASE),
         search_in_file_func=search_in_file_func,

scripts/compare/.gitignore

@@ -0,0 +1 @@
+/results/

scripts/compare/compare.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2024 IOTA Stiftung
+# SPDX-License-Identifier: Apache-2.0
+import os, sys, subprocess, argparse, tempfile
+sys.path.append('../utils')
+from codeowners import CodeOwners
+
+# create symlinks in target for the matching folders of the code owners
+def create_symlinks(source, target, code_owners, owners):
+    # check if the target directory already exists
+    if os.path.exists(target):
+        raise Exception(f"Target directory '{target}' already exists.")
+
+    # get matching folders for the codeowners
+    matching_paths = code_owners.match_paths_for_owners(source, owners)
+    if not matching_paths:
+        raise Exception("No matching folders found for the code owners.")
+
+    # create target directory
+    os.makedirs(target)
+
+    # create symlinks for the matching folders
+    for path in matching_paths:
+        source_path = os.path.abspath(os.path.join(source, path))
+        target_path = os.path.abspath(os.path.join(target, path))
+        os.makedirs(os.path.dirname(target_path), exist_ok=True)
+        os.symlink(source_path, target_path, target_is_directory=True)
+
+    print(f"Symlinks created in '{target}'.")
+
+# open tool for comparison
+def run_compare_tool(compare_tool_binary, compare_tool_arguments, source_folder, target_folder):
+    print(f"Opening {compare_tool_binary} for comparison between {source_folder} and {target_folder}...")
+
+    cmd = [compare_tool_binary]
+    if compare_tool_arguments:
+        cmd = cmd + compare_tool_arguments.split(" ")
+    cmd = cmd + [source_folder, target_folder]
+
+    subprocess.run(cmd)
+
+################################################################################
+if __name__ == "__main__":
+    # Argument parser setup
+    parser = argparse.ArgumentParser(description="Repository comparison tool with optional code ownership filter.")
+    parser.add_argument('--source-folder', required=True, help="The path to the source folder for comparison.")
+    parser.add_argument('--target-folder', required=True, help="The path to the target folder for comparison.")
+    parser.add_argument('--codeowners', nargs='+', help='Optionally filter folders by code owners (e.g., "@iotaledger/node @iotaledger/consensus").')
+    parser.add_argument('--codeowners-file', default="", help="The path to the code owners file.")
+    parser.add_argument('--compare-tool-binary', default="meld", help="The binary to use for comparison.")
+    parser.add_argument('--compare-tool-arguments', default="", help="The arguments to use for comparison.")
+    args = parser.parse_args()
+
+    # get the folder paths
+    source_folder = os.path.abspath(os.path.expanduser(args.source_folder))
+    target_folder = os.path.abspath(os.path.expanduser(args.target_folder))
+
+    # check if we want to filter by code owners
+    if args.codeowners:
+        # get the folder the script is in
+        script_folder = os.path.dirname(os.path.realpath(__file__))
+
+        # load the code owners file
+        code_owners_file = os.path.join(script_folder, "..", "..", ".github", "CODEOWNERS")
+        if args.codeowners_file != "":
+            code_owners_file = os.path.abspath(os.path.expanduser(args.codeowners_file))
+
+        code_owners = CodeOwners(code_owners_file)
+
+        results = os.path.abspath(os.path.join(script_folder, "results"))
+        os.makedirs(results, exist_ok=True)
+
+        # Create temporary root folder for the target with random name suffix
+        with tempfile.TemporaryDirectory(prefix="source_", dir=results) as temp_dir_source:
+            create_symlinks(source_folder, os.path.join(temp_dir_source, "main"), code_owners, args.codeowners)
+
+            with tempfile.TemporaryDirectory(prefix="target_", dir=results) as temp_dir_target:
+                create_symlinks(target_folder, os.path.join(temp_dir_target, "main"), code_owners, args.codeowners)
+
+                run_compare_tool(args.compare_tool_binary, args.compare_tool_arguments, temp_dir_source, temp_dir_target)
+    else:
+        run_compare_tool(args.compare_tool_binary, args.compare_tool_arguments, source_folder, target_folder)

scripts/compare/run.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# usage: compare.py [-h] 
+#                   --source-folder SOURCE_FOLDER
+#                   --target-folder TARGET_FOLDER
+#                   [--codeowners CODEOWNERS [CODEOWNERS ...]]
+#                   [--codeowners-file CODEOWNERS_FILE]
+#                   [--compare-tool-binary COMPARE_TOOL_BINARY]
+#                   [--compare-tool-arguments COMPARE_TOOL_ARGUMENTS]
+# 
+# Repository comparison tool with optional code ownership filter.
+# 
+# options:
+#   -h, --help            show this help message and exit
+#   --source-folder SOURCE_FOLDER
+#                         The path to the source folder for comparison.
+#   --target-folder TARGET_FOLDER
+#                         The path to the target folder for comparison.
+#   --codeowners CODEOWNERS [CODEOWNERS ...]
+#                         Optionally filter folders by code owners (e.g., "@iotaledger/node @iotaledger/consensus").
+#   --codeowners-file CODEOWNERS_FILE
+#                         The path to the code owners file.
+#   --compare-tool-binary COMPARE_TOOL_BINARY
+#                         The binary to use for comparison.
+#   --compare-tool-arguments COMPARE_TOOL_ARGUMENTS
+#                         The arguments to use for comparison.
+source ../utils/python_venv_wrapper.sh
+
+$PYTHON_CMD compare.py \
+    --source-folder ../slipstream/results/mainnet-v1.32.2/main \
+    --target-folder ../slipstream/results/mainnet-v1.41.1/main \
+    --codeowners @iotaledger/node \
+    "$@"

scripts/create_patches/.gitignore

@@ -0,0 +1 @@
+/patches/