From 9a6f2b5d50d2d325e01a1f6ac08c7146b04605c2 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 11:32:42 +0100 Subject: [PATCH 01/13] scripts: Move `python_venv_wrapper.sh` to `scripts/utils/` --- scripts/codesearch/python_venv_wrapper.sh | 47 ----------------- scripts/codesearch/run_search_tests.sh | 2 +- scripts/codesearch/run_search_versions.sh | 2 +- .../dependency_graphs/python_venv_wrapper.sh | 47 ----------------- scripts/dependency_graphs/run.sh | 2 +- scripts/slipstream/python_venv_wrapper.sh | 47 ----------------- scripts/slipstream/run_slipstream.sh | 2 +- .../python_venv_wrapper.sh | 45 ---------------- scripts/track_upstream_commits/run.sh | 2 +- scripts/utils/python_venv_wrapper.sh | 52 +++++++++++++++++++ 10 files changed, 57 insertions(+), 191 deletions(-) delete mode 100644 scripts/codesearch/python_venv_wrapper.sh delete mode 100644 scripts/dependency_graphs/python_venv_wrapper.sh delete mode 100644 scripts/slipstream/python_venv_wrapper.sh delete mode 100644 scripts/track_upstream_commits/python_venv_wrapper.sh create mode 100644 scripts/utils/python_venv_wrapper.sh diff --git a/scripts/codesearch/python_venv_wrapper.sh b/scripts/codesearch/python_venv_wrapper.sh deleted file mode 100644 index ce65e312017..00000000000 --- a/scripts/codesearch/python_venv_wrapper.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Deactivate the virtual environment if it was activated -cleanup() { - if [ "$in_venv" = true ]; then - echo "Deactivating virtual environment..." - deactivate - fi -} - -# Set a trap to call cleanup on exit or error -trap cleanup EXIT - -# Check if either "python" or "python3" exists and use it -if command -v python3 &>/dev/null; then - PYTHON_CMD="python3" -elif command -v python &>/dev/null; then - PYTHON_CMD="python" -else - echo "Neither python nor python3 binary is installed. Please install Python." - exit 1 -fi - -# Check if running in a virtual environment -if [ -z "$VIRTUAL_ENV" ]; then - echo "Not in a virtual environment. Activating..." - - # Check if the "venv" folder and venv/bin/activate exists - if [ ! -f "venv/bin/activate" ]; then - # Delete a potentially broken venv folder - if [ -d "venv" ]; then - echo "Deleting broken virtual environment..." - rm -rf venv - fi - - echo "Virtual environment doesn't exist yet, creating..." - - $PYTHON_CMD -m venv venv - source venv/bin/activate - - echo "Installing packages..." - pip install -r requirements.txt - else - source venv/bin/activate - in_venv=true - fi -fi diff --git a/scripts/codesearch/run_search_tests.sh b/scripts/codesearch/run_search_tests.sh index 8a6562214de..7d71b10ae8c 100755 --- a/scripts/codesearch/run_search_tests.sh +++ b/scripts/codesearch/run_search_tests.sh @@ -5,6 +5,6 @@ # --output # Output file to save the results. # --verbose # Display detailed file path and line number for each occurrence. # --debug # Display the line where the occurrence was found. -source python_venv_wrapper.sh +source ../utils/python_venv_wrapper.sh $PYTHON_CMD search_tests.py --regex "zklogin" --verbose --debug \ No newline at end of file diff --git a/scripts/codesearch/run_search_versions.sh b/scripts/codesearch/run_search_versions.sh index 5edfd48e3de..ed0e1d06e70 100755 --- a/scripts/codesearch/run_search_versions.sh +++ b/scripts/codesearch/run_search_versions.sh @@ -4,6 +4,6 @@ # --output # Output file to save the results. # --verbose # Display detailed file path and line number for each occurrence. # --debug # Display the line where the occurrence was found. -source python_venv_wrapper.sh +source ../utils/python_venv_wrapper.sh $PYTHON_CMD search_versions.py --verbose #--debug \ No newline at end of file diff --git a/scripts/dependency_graphs/python_venv_wrapper.sh b/scripts/dependency_graphs/python_venv_wrapper.sh deleted file mode 100644 index ce65e312017..00000000000 --- a/scripts/dependency_graphs/python_venv_wrapper.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Deactivate the virtual environment if it was activated -cleanup() { - if [ "$in_venv" = true ]; then - echo "Deactivating virtual environment..." - deactivate - fi -} - -# Set a trap to call cleanup on exit or error -trap cleanup EXIT - -# Check if either "python" or "python3" exists and use it -if command -v python3 &>/dev/null; then - PYTHON_CMD="python3" -elif command -v python &>/dev/null; then - PYTHON_CMD="python" -else - echo "Neither python nor python3 binary is installed. Please install Python." - exit 1 -fi - -# Check if running in a virtual environment -if [ -z "$VIRTUAL_ENV" ]; then - echo "Not in a virtual environment. Activating..." - - # Check if the "venv" folder and venv/bin/activate exists - if [ ! -f "venv/bin/activate" ]; then - # Delete a potentially broken venv folder - if [ -d "venv" ]; then - echo "Deleting broken virtual environment..." - rm -rf venv - fi - - echo "Virtual environment doesn't exist yet, creating..." - - $PYTHON_CMD -m venv venv - source venv/bin/activate - - echo "Installing packages..." - pip install -r requirements.txt - else - source venv/bin/activate - in_venv=true - fi -fi diff --git a/scripts/dependency_graphs/run.sh b/scripts/dependency_graphs/run.sh index 0f1b7ed729a..b74881a08b5 100755 --- a/scripts/dependency_graphs/run.sh +++ b/scripts/dependency_graphs/run.sh @@ -2,6 +2,6 @@ # OPTIONS: # --target-folder # The path to the target folder. # --skip-dev-dependencies # Whether or not to include the `dev-dependencies`. -source python_venv_wrapper.sh +source ../utils/python_venv_wrapper.sh $PYTHON_CMD dependency_graphs.py --skip-dev-dependencies \ No newline at end of file diff --git a/scripts/slipstream/python_venv_wrapper.sh b/scripts/slipstream/python_venv_wrapper.sh deleted file mode 100644 index ce65e312017..00000000000 --- a/scripts/slipstream/python_venv_wrapper.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Deactivate the virtual environment if it was activated -cleanup() { - if [ "$in_venv" = true ]; then - echo "Deactivating virtual environment..." - deactivate - fi -} - -# Set a trap to call cleanup on exit or error -trap cleanup EXIT - -# Check if either "python" or "python3" exists and use it -if command -v python3 &>/dev/null; then - PYTHON_CMD="python3" -elif command -v python &>/dev/null; then - PYTHON_CMD="python" -else - echo "Neither python nor python3 binary is installed. Please install Python." - exit 1 -fi - -# Check if running in a virtual environment -if [ -z "$VIRTUAL_ENV" ]; then - echo "Not in a virtual environment. Activating..." - - # Check if the "venv" folder and venv/bin/activate exists - if [ ! -f "venv/bin/activate" ]; then - # Delete a potentially broken venv folder - if [ -d "venv" ]; then - echo "Deleting broken virtual environment..." - rm -rf venv - fi - - echo "Virtual environment doesn't exist yet, creating..." - - $PYTHON_CMD -m venv venv - source venv/bin/activate - - echo "Installing packages..." - pip install -r requirements.txt - else - source venv/bin/activate - in_venv=true - fi -fi diff --git a/scripts/slipstream/run_slipstream.sh b/scripts/slipstream/run_slipstream.sh index 164e92da213..f1d4ae3e924 100755 --- a/scripts/slipstream/run_slipstream.sh +++ b/scripts/slipstream/run_slipstream.sh @@ -30,7 +30,7 @@ # --compare-source-folder FOLDER # The path to the source folder for comparison. # --compare-tool-binary BINARY # The binary to use for comparison. # --compare-tool-arguments ARGUMENTS # The arguments to use for comparison. -source python_venv_wrapper.sh +source ../utils/python_venv_wrapper.sh #REPO_TAG=mainnet-v1.29.2 #REPO_TAG=mainnet-v1.32.2 diff --git a/scripts/track_upstream_commits/python_venv_wrapper.sh b/scripts/track_upstream_commits/python_venv_wrapper.sh deleted file mode 100644 index 4ec7ec5e13b..00000000000 --- a/scripts/track_upstream_commits/python_venv_wrapper.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Deactivate the virtual environment if it was activated -cleanup() { - if [ "$in_venv" = true ]; then - echo "Deactivating virtual environment..." - deactivate - fi -} - -# Set a trap to call cleanup on exit or error -trap cleanup EXIT - -# Check if either "python" or "python3" exists and use it -if command -v python3 &>/dev/null; then - PYTHON_CMD="python3" -elif command -v python &>/dev/null; then - PYTHON_CMD="python" -else - echo "Neither python nor python3 binary is installed. Please install Python." - exit 1 -fi - -# Check if running in a virtual environment -if [ -z "$VIRTUAL_ENV" ]; then - echo "Not in a virtual environment. Activating..." - - # Check if the "venv" folder and venv/bin/activate exists - if [ ! -f "venv/bin/activate" ]; then - # Delete a potentially broken venv folder - if [ -d "venv" ]; then - echo "Deleting broken virtual environment..." - rm -rf venv - fi - - echo "Virtual environment doesn't exist yet, creating..." - - $PYTHON_CMD -m venv venv - source venv/bin/activate - - else - source venv/bin/activate - in_venv=true - fi -fi diff --git a/scripts/track_upstream_commits/run.sh b/scripts/track_upstream_commits/run.sh index 895c544228e..ecdd41ebd5b 100755 --- a/scripts/track_upstream_commits/run.sh +++ b/scripts/track_upstream_commits/run.sh @@ -13,7 +13,7 @@ # --target-folder TARGET_FOLDER # The path to the target folder. # --clone-source Clone the upstream repository. -source python_venv_wrapper.sh +source ../utils/python_venv_wrapper.sh $PYTHON_CMD track_upstream_commits.py \ --repo-tag "mainnet-v1.36.2" \ diff --git a/scripts/utils/python_venv_wrapper.sh b/scripts/utils/python_venv_wrapper.sh new file mode 100644 index 00000000000..99a1f2c3558 --- /dev/null +++ b/scripts/utils/python_venv_wrapper.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Deactivate the virtual environment if it was activated +cleanup() { + if [ "$in_venv" = true ]; then + echo "Deactivating virtual environment..." + deactivate + fi +} + +# Set a trap to call cleanup on exit or error +trap cleanup EXIT + +# Check if either "python" or "python3" exists and use it +if command -v python3 &>/dev/null; then + PYTHON_CMD="python3" +elif command -v python &>/dev/null; then + PYTHON_CMD="python" +else + echo "Neither python nor python3 binary is installed. Please install Python." + exit 1 +fi + +# Check if the "requirements.txt" file exists +if [ -f "requirements.txt" ]; then + # Only enter a virtual environment if there are dependencies + + # Check if running in a virtual environment + if [ -z "$VIRTUAL_ENV" ]; then + echo "Not in a virtual environment. Activating..." + + # Check if the "venv" folder and venv/bin/activate exists + if [ ! -f "venv/bin/activate" ]; then + # Delete a potentially broken venv folder + if [ -d "venv" ]; then + echo "Deleting broken virtual environment..." + rm -rf venv + fi + + echo "Virtual environment doesn't exist yet, creating..." + + $PYTHON_CMD -m venv venv + source venv/bin/activate + + echo "Installing packages..." + pip install -r requirements.txt + else + source venv/bin/activate + in_venv=true + fi + fi +fi From d06e96b44a70e5aa56c793d10ff3d91503d7d333 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 11:33:11 +0100 Subject: [PATCH 02/13] scripts: add `__pycache__` to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e58897c6432..7013bcbc7ce 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ Move.lock # Python .pyc +**/__pycache__/ # Node.js node_modules/ From 185dd1e992320feb57c0b68745d624d8c30e347b Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 11:35:17 +0100 Subject: [PATCH 03/13] scripts: Move bash print functions to `scripts/utils/common.sh` --- docker/utils/build-script.sh | 13 ++++------- .../generate_files/update_generated_files.sh | 23 +++++++------------ scripts/update_light_client_checkpoints.sh | 17 ++++---------- {docker => scripts}/utils/common.sh | 0 4 files changed, 18 insertions(+), 35 deletions(-) rename {docker => scripts}/utils/common.sh (100%) diff --git a/docker/utils/build-script.sh b/docker/utils/build-script.sh index 91903fc8db6..4f4b3338cd2 100755 --- a/docker/utils/build-script.sh +++ b/docker/utils/build-script.sh @@ -3,19 +3,16 @@ # Modifications Copyright (c) 2024 IOTA Stiftung # SPDX-License-Identifier: Apache-2.0 -# Get the directory where build-script.sh is located -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# Source common.sh from the utils directory -source "$SCRIPT_DIR/common.sh" - # fast fail. set -e +REPO_ROOT="$(git rev-parse --show-toplevel)" + +# Source common.sh from the utils directory +source "$REPO_ROOT/scripts/utils/common.sh" + # Get the current working directory where the script was called CURRENT_WORKING_DIR="$(pwd)" - -REPO_ROOT="$(git rev-parse --show-toplevel)" DOCKERFILE="$CURRENT_WORKING_DIR/Dockerfile" GIT_REVISION="$(git describe --always --abbrev=12 --dirty --exclude '*')" BUILD_DATE="$(date -u +'%Y-%m-%d')" diff --git a/scripts/generate_files/update_generated_files.sh b/scripts/generate_files/update_generated_files.sh index 15aadafc962..510171ae7ae 100755 --- a/scripts/generate_files/update_generated_files.sh +++ b/scripts/generate_files/update_generated_files.sh @@ -4,6 +4,14 @@ SKIP_SPEC_GENERATION=false SKIP_TS_GENERATION=false CHECK_BUILDS=false +# fast fail. +set -e + +REPO_ROOT="$(git rev-parse --show-toplevel)" + +# Source common.sh from the utils directory +source "$REPO_ROOT/scripts/utils/common.sh" + # Parse command line arguments # Usage: # --target-folder - the target folder of the repository @@ -40,21 +48,6 @@ done # Resolve the target folder TARGET_FOLDER=$(realpath ${TARGET_FOLDER}) -function print_step { - echo -e "\e[32m$1\e[0m" -} - -function print_error { - echo -e "\e[31m$1\e[0m" -} - -function check_error { - if [ $? -ne 0 ]; then - print_error "$1" - exit 1 - fi -} - function docker_run { docker run --rm --name pnpm-cargo-image -v ${TARGET_FOLDER}:/home/node/app:rw --user $(id -u):$(id -g) pnpm-cargo-image sh -c "$1" } diff --git a/scripts/update_light_client_checkpoints.sh b/scripts/update_light_client_checkpoints.sh index ec75995a7f6..659290677d3 100755 --- a/scripts/update_light_client_checkpoints.sh +++ b/scripts/update_light_client_checkpoints.sh @@ -2,20 +2,13 @@ # Copyright (c) 2024 IOTA Stiftung # SPDX-License-Identifier: Apache-2.0 -function print_step { - echo -e "\e[32m$1\e[0m" -} +# fast fail. +set -e -function print_error { - echo -e "\e[31m$1\e[0m" -} +REPO_ROOT="$(git rev-parse --show-toplevel)" -function check_error { - if [ $? -ne 0 ]; then - print_error "$1" - exit 1 - fi -} +# Source common.sh from the utils directory +source "$REPO_ROOT/scripts/utils/common.sh" # Change to the root directory of the repository pushd "../" diff --git a/docker/utils/common.sh b/scripts/utils/common.sh similarity index 100% rename from docker/utils/common.sh rename to scripts/utils/common.sh From bfb471786a4d66a25e392fc36251b985f8980e79 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 11:36:12 +0100 Subject: [PATCH 04/13] scripts: add more `ignored_dirs` to codesearch scripts --- scripts/codesearch/search_tests.py | 12 ++++++++++-- scripts/codesearch/search_versions.py | 15 ++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/scripts/codesearch/search_tests.py b/scripts/codesearch/search_tests.py index c5453cf6bc4..fd7e3a105ea 100644 --- a/scripts/codesearch/search_tests.py +++ b/scripts/codesearch/search_tests.py @@ -108,10 +108,18 @@ def merge_results_func(results, result): results = search_files_in_parallel( target_dir=args.target, ignored_dirs=[ + '.cargo', + '.changeset', + '.config', '.git', - 'scripts', - 'node_modules', + '.github', + '.husky', '.pnpm_store', + '.vscode', + 'chocolatey', + 'linting', + 'scripts', + 'target', ], file_pattern=re.compile(r'\.rs$', flags=re.IGNORECASE), search_in_file_func=search_in_file_func, diff --git a/scripts/codesearch/search_versions.py b/scripts/codesearch/search_versions.py index 1bbdee9874c..8a3c20db243 100644 --- a/scripts/codesearch/search_versions.py +++ b/scripts/codesearch/search_versions.py @@ -76,11 +76,16 @@ def merge_results_func(results, result): results = search_files_in_parallel( target_dir=args.target, ignored_dirs=[ + '.cargo', + '.changeset', + '.config', '.git', - 'scripts', - 'node_modules', + '.github', + '.husky', '.pnpm_store', - 'unit_tests', + '.vscode', + 'chocolatey', + 'linting', 'move-compiler', 'move-vm-types', 'move-bytecode-verifier', @@ -90,6 +95,10 @@ def merge_results_func(results, result): 'move-model', 'move-prover', 'move-vm-integration-tests', + 'node_modules', + 'scripts', + 'target', + 'unit_tests', ], file_pattern=re.compile(r'\.rs$', flags=re.IGNORECASE), search_in_file_func=search_in_file_func, From 30df71d48fc5c6fec83b0b543df9daeb862ccbb0 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 11:50:25 +0100 Subject: [PATCH 05/13] scripts: make slipstream compatible with 1.41.1 --- scripts/slipstream/config_slipstream.json | 7 ++++++ scripts/slipstream/patches/copyright.patch | 25 ------------------- .../patches/v1.41.0/copyright.patch | 25 +++++++++++++++++++ scripts/slipstream/run_slipstream.sh | 11 +++++--- 4 files changed, 40 insertions(+), 28 deletions(-) create mode 100644 scripts/slipstream/patches/v1.41.0/copyright.patch diff --git a/scripts/slipstream/config_slipstream.json b/scripts/slipstream/config_slipstream.json index 5b3e478bb68..33de07c0cd8 100644 --- a/scripts/slipstream/config_slipstream.json +++ b/scripts/slipstream/config_slipstream.json @@ -992,6 +992,13 @@ "revert": false, "comment": "we want to have our copyright" }, + "copyright_v1.41.0": { + "path": "patches/v1.41.0/copyright.patch", + "min_sem_version": "", + "max_sem_version": "1.41.0", + "revert": false, + "comment": "we want to have our copyright" + }, "gitignore": { "path": "patches/gitignore.patch", "min_sem_version": "", diff --git a/scripts/slipstream/patches/copyright.patch b/scripts/slipstream/patches/copyright.patch index ed90ebc0e12..b89473885e1 100644 --- a/scripts/slipstream/patches/copyright.patch +++ b/scripts/slipstream/patches/copyright.patch @@ -60,28 +60,3 @@ index e79379d..80d7345 100644 -diff --git a/sdk/typescript/genversion.mjs b/sdk/typescript/genversion.mjs -index c5daa09..645895d 100644 ---- a/sdk/typescript/genversion.mjs -+++ b/sdk/typescript/genversion.mjs -@@ -4,7 +4,7 @@ - import { readFile, writeFile } from 'fs/promises'; - import TOML from '@iarna/toml'; - --const LICENSE = '// Copyright (c) Mysten Labs, Inc.\n// SPDX-License-Identifier: Apache-2.0\n\n'; -+const LICENSE = '// Copyright (c) Mysten Labs, Inc.\n// Modifications Copyright (c) 2024 IOTA Stiftung\n// SPDX-License-Identifier: Apache-2.0\n\n'; - - const WARNING = '// This file is generated by genversion.mjs. Do not edit it directly.\n\n'; - -diff --git a/crates/iota-move/scripts/prover_setup.sh b/crates/iota-move/scripts/prover_setup.sh -index da0ec56..35ab871 100755 ---- a/crates/iota-move/scripts/prover_setup.sh -+++ b/crates/iota-move/scripts/prover_setup.sh -@@ -1,7 +1,4 @@ - # Copyright (c) Mysten Labs, Inc. --# Modifications Copyright (c) 2024 IOTA Stiftung --# SPDX-License-Identifier: Apache-2.0 -- - # Copyright (c) The Diem Core Contributors - # Copyright (c) The Move Contributors - # Modifications Copyright (c) 2024 IOTA Stiftung diff --git a/scripts/slipstream/patches/v1.41.0/copyright.patch b/scripts/slipstream/patches/v1.41.0/copyright.patch new file mode 100644 index 00000000000..0656daab971 --- /dev/null +++ b/scripts/slipstream/patches/v1.41.0/copyright.patch @@ -0,0 +1,25 @@ +diff --git a/sdk/typescript/genversion.mjs b/sdk/typescript/genversion.mjs +index c5daa09..645895d 100644 +--- a/sdk/typescript/genversion.mjs ++++ b/sdk/typescript/genversion.mjs +@@ -4,7 +4,7 @@ + import { readFile, writeFile } from 'fs/promises'; + import TOML from '@iarna/toml'; + +-const LICENSE = '// Copyright (c) Mysten Labs, Inc.\n// SPDX-License-Identifier: Apache-2.0\n\n'; ++const LICENSE = '// Copyright (c) Mysten Labs, Inc.\n// Modifications Copyright (c) 2024 IOTA Stiftung\n// SPDX-License-Identifier: Apache-2.0\n\n'; + + const WARNING = '// This file is generated by genversion.mjs. Do not edit it directly.\n\n'; + +diff --git a/crates/iota-move/scripts/prover_setup.sh b/crates/iota-move/scripts/prover_setup.sh +index da0ec56..35ab871 100755 +--- a/crates/iota-move/scripts/prover_setup.sh ++++ b/crates/iota-move/scripts/prover_setup.sh +@@ -1,7 +1,4 @@ + # Copyright (c) Mysten Labs, Inc. +-# Modifications Copyright (c) 2024 IOTA Stiftung +-# SPDX-License-Identifier: Apache-2.0 +- + # Copyright (c) The Diem Core Contributors + # Copyright (c) The Move Contributors + # Modifications Copyright (c) 2024 IOTA Stiftung diff --git a/scripts/slipstream/run_slipstream.sh b/scripts/slipstream/run_slipstream.sh index f1d4ae3e924..33a676e059c 100755 --- a/scripts/slipstream/run_slipstream.sh +++ b/scripts/slipstream/run_slipstream.sh @@ -35,7 +35,8 @@ source ../utils/python_venv_wrapper.sh #REPO_TAG=mainnet-v1.29.2 #REPO_TAG=mainnet-v1.32.2 #REPO_TAG=mainnet-v1.36.2 -REPO_TAG=mainnet-v1.40.3 +#REPO_TAG=mainnet-v1.40.3 +REPO_TAG=mainnet-v1.41.1 $PYTHON_CMD slipstream.py \ --config config_slipstream.json \ @@ -53,6 +54,10 @@ $PYTHON_CMD slipstream.py \ --run-fix-typos \ --run-cargo-fmt \ --run-dprint-fmt \ - --run-pnpm-prettier-fix \ - --run-pnpm-lint-fix \ --run-cargo-clippy + + # we ignore these steps from now on, because with 1.41.1 the ts-sdk was moved out, + # and with 1.43.0 the apps folder as well. For older versions than 1.41.1, + # these can be reenabled if needed. + #--run-pnpm-prettier-fix \ + #--run-pnpm-lint-fix \ From f5a83976144e5b07f4fa37baef888108892422dd Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 15:32:39 +0100 Subject: [PATCH 06/13] scripts: refactor out `CodeOwners` and move to utils --- .../dependency_graphs/dependency_graphs.py | 51 ++--- scripts/track_upstream_commits/README.md | 25 ++- .../track_upstream_commits.py | 112 +++++------ scripts/utils/codeowners.py | 185 ++++++++++++++++++ 4 files changed, 270 insertions(+), 103 deletions(-) create mode 100644 scripts/utils/codeowners.py diff --git a/scripts/dependency_graphs/dependency_graphs.py b/scripts/dependency_graphs/dependency_graphs.py index 21c764deb00..9d6e871c232 100644 --- a/scripts/dependency_graphs/dependency_graphs.py +++ b/scripts/dependency_graphs/dependency_graphs.py @@ -3,39 +3,16 @@ # This script creates dependency graphs between the internal crates. # It outputs SVG files and an index.html. - -import re, pathlib, os, io, subprocess, argparse +import re, sys, pathlib, os, io, subprocess, argparse from bs4 import BeautifulSoup - -# holds all possible codeowners -class CodeOwners(object): - def __init__(self, file_path): - self.codeowners = {} - with open(file_path, 'r') as file: - for line in file: - line = line.strip() - if line and not line.startswith('#'): - pattern, *owners = line.split() - self.codeowners[pattern] = owners - - def get_code_owner(self, crate_path): - for pattern, owners in self.codeowners.items(): - if pattern == "*": - # skip the fallback here, we want to check all other patterns first - continue - - # Check if the pattern matches the relative path of the crate - regex_pattern = '^' + re.escape(pattern).replace(r'\*', '.*') - if re.match(regex_pattern, crate_path) or re.match(regex_pattern, crate_path+'/'): - return ", ".join(owners) - - return ", ".join(self.codeowners.get('*', ["No owners specified"])) +sys.path.append('../utils') +from codeowners import CodeOwners # holds infos about a crate class Crate(object): - def __init__(self, name, owner): + def __init__(self, name, owners): self.name = name - self.owner = owner + self.owners = owners self.dependencies = {} # holds infos about a dependency @@ -80,9 +57,9 @@ def get_internal_package_info(base_path, line): def get_crate_name_by_package_info(package_info): return re.sub(r' v[0-9]+\.[0-9]+\.[0-9]+.*', '', package_info).strip() -def get_code_owner_by_package_info(code_owners, base_path, package_info): +def get_code_owners_by_package_info(code_owners, base_path, package_info): crate_path = re.search(r'\((.*?)\)', package_info).group(1).replace(str(base_path), '') - return code_owners.get_code_owner(crate_path) + return ", ".join(code_owners.match_owners_for_path(crate_path)) # parse the cargo tree from a file into a map of dependencies def parse_cargo_tree(cargo_tree_output, base_path, code_owners, skip_dev_dependencies): @@ -100,9 +77,9 @@ def parse_cargo_tree(cargo_tree_output, base_path, code_owners, skip_dev_depende continue crate_name = get_crate_name_by_package_info(package_info) - code_owner = get_code_owner_by_package_info(code_owners, base_path, package_info) + code_owners_line = get_code_owners_by_package_info(code_owners, base_path, package_info) - crates_dict[crate_name] = Crate(crate_name, code_owner) + crates_dict[crate_name] = Crate(crate_name, code_owners_line) # loop again to determine the dependencies crate_names_stack = [] @@ -198,13 +175,13 @@ def generate_dot_all(output_folder, crates_dict): for crate in crates_dict.values(): if crate.name not in visited: visited[crate.name] = True - buffer.write(f' "{crate.name}" [label="{crate.name}\n({crate.owner})"];\n') + buffer.write(f' "{crate.name}" [label="{crate.name}\n({crate.owners})"];\n') for dependency in crate.dependencies.values(): dependency_crate = crates_dict[dependency.name] if dependency_crate.name not in visited: visited[dependency_crate.name] = True - buffer.write(f' "{dependency_crate.name}" [label="{dependency_crate.name}\n({dependency_crate.owner})", fontcolor={get_node_color(crates_dict, dependency)}];\n') + buffer.write(f' "{dependency_crate.name}" [label="{dependency_crate.name}\n({dependency_crate.owners})", fontcolor={get_node_color(crates_dict, dependency)}];\n') buffer.write(f' "{crate.name}" -> "{dependency_crate.name}";\n') @@ -225,19 +202,19 @@ def generate_dot_per_crate(output_folder, crates_dict, traverse_all): buffer = io.StringIO() # Set the text color of the current crate to green - buffer.write(f' "{crate.name}" [label="{crate.name}\n({crate.owner})", fontcolor=green];\n') + buffer.write(f' "{crate.name}" [label="{crate.name}\n({crate.owners})", fontcolor=green];\n') # Add reverse dependencies if crate.name in reverse_dependency_names: for reverse_parent_name in reverse_dependency_names[crate.name]: parent_crate = crates_dict[reverse_parent_name] - buffer.write(f' "{parent_crate.name}" [label="{parent_crate.name}\n({parent_crate.owner})", fontcolor=blue];\n') + buffer.write(f' "{parent_crate.name}" [label="{parent_crate.name}\n({parent_crate.owners})", fontcolor=blue];\n') buffer.write(f' "{parent_crate.name}" -> "{crate.name}";\n') def write_crate_dependencies(crate, traversed_dict): for dependency in crate.dependencies.values(): dependency_crate = crates_dict[dependency.name] - buffer.write(f' "{dependency_crate.name}" [label="{dependency_crate.name}\n({dependency_crate.owner})", fontcolor={get_node_color(crates_dict, dependency)}];\n') + buffer.write(f' "{dependency_crate.name}" [label="{dependency_crate.name}\n({dependency_crate.owners})", fontcolor={get_node_color(crates_dict, dependency)}];\n') buffer.write(f' "{crate.name}" -> "{dependency_crate.name}";\n') if traverse_all: diff --git a/scripts/track_upstream_commits/README.md b/scripts/track_upstream_commits/README.md index e982304bc72..fa3d9d12987 100644 --- a/scripts/track_upstream_commits/README.md +++ b/scripts/track_upstream_commits/README.md @@ -5,21 +5,26 @@ This script identifies all commits for the folders managed by the specified code ## Usage ```bash -usage: track_upstream_commits.py [-h] --since SINCE --until UNTIL [--folders FOLDERS [FOLDERS ...]] - [--codeowner CODEOWNER] [--repo-url REPO_URL] [--repo-tag REPO_TAG] - [--target-folder TARGET_FOLDER] [--clone-source] +usage: track_upstream_commits.py [-h] + --since SINCE + --until UNTIL + [--codeowners CODEOWNERS [CODEOWNERS ...]] + [--folders FOLDERS [FOLDERS ...]] + [--repo-url REPO_URL] + [--repo-tag REPO_TAG] + [--target-folder TARGET_FOLDER] + [--clone-source] Track upstream commits for specified folders. options: -h, --help show this help message and exit - --since SINCE Start commit hash or the tag for git log (e.g., "bb778828e36d53a7d91a27e55109f2f45621badc", "mainnet-v1.32.2"), it is EXCLUDED from the - results. + --since SINCE Start commit hash or the tag for git log (e.g., "bb778828e36d53a7d91a27e55109f2f45621badc", "mainnet-v1.32.2"), it is EXCLUDED from the results. --until UNTIL End commit hash or the tag for git log (e.g., "3ada97c109cc7ae1b451cb384a1f2cfae49c8d3e", "mainnet-v1.36.2"), it is INCLUDED in the results. + --codeowners CODEOWNERS [CODEOWNERS ...] + Code owners of the folders (e.g., "@iotaledger/node @iotaledger/consensus"). --folders FOLDERS [FOLDERS ...] List of folders relative to the project root to track (e.g., "crates/iota-core crates/iota-node"). - --codeowner CODEOWNER - code owner of the folders (e.g., "node") --repo-url REPO_URL The URL to the repository. Can also be a local folder. --repo-tag REPO_TAG The tag to checkout in the repository. --target-folder TARGET_FOLDER @@ -35,12 +40,12 @@ input: ```bash ./run.sh --since bb778828e36d53a7d91a27e55109f2f45621badc --until 3ada97c109cc7ae1b451cb384a1f2cfae49c8d3e --crates crates/iota-bridge --co -deowner node +codeowners @iotaledger/node ``` output: -The results include the `crates/iota-bridge` and all the folders that are managed by the `node` team. +The results include the `crates/iota-bridge` and all the folders that are managed by the `@iotaledger/node` team. ``` Not in a virtual environment. Activating... @@ -78,5 +83,5 @@ CRATES: crates/sui-bridge, docker, crates/sui-archival, crates/sui-authority-agg Or use it with tags: ``` -./run.sh --since mainnet-v1.32.2 --until mainnet-v1.36.2 --codeowner node +./run.sh --since mainnet-v1.32.2 --until mainnet-v1.36.2 --codeowners @iotaledger/node ``` diff --git a/scripts/track_upstream_commits/track_upstream_commits.py b/scripts/track_upstream_commits/track_upstream_commits.py index fa6ccbbd7f5..cc63faf42d7 100644 --- a/scripts/track_upstream_commits/track_upstream_commits.py +++ b/scripts/track_upstream_commits/track_upstream_commits.py @@ -1,9 +1,8 @@ -import subprocess -import os -import argparse -import re -import pathlib -import shutil +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 +import os, sys, subprocess, argparse, re, pathlib, shutil +sys.path.append('../utils') +from codeowners import CodeOwners def is_commit_hash(ref): # A commit hash is a 40-character hexadecimal string @@ -55,7 +54,7 @@ def is_following_branch(): # Change working directory to the cloned repo os.chdir(target_folder) -def convert_iota_to_sui_folders(folders): +def iota_to_sui_mapping_func(str): premap = { 'crates/iota-common': 'crates/mysten-common', 'crates/iota-metrics': 'crates/mysten-metrics', @@ -64,38 +63,11 @@ def convert_iota_to_sui_folders(folders): 'crates/iota-util-mem-derive': 'crates/mysten-util-mem-derive', } - mapped_folders = [] - for folder in folders: - if folder in premap: - folder = premap[folder] - mapped_folders.append(folder.replace('iota', 'sui')) - - return mapped_folders - -# Parse the CODEOWNERS file and return the folders of the code owner -def get_folders_for_code_owner(file_path, code_owner): - folder_to_owners = {} - matched_folders = [] - with open(file_path, 'r') as file: - for line in file: - line = line.strip() - if line and not line.startswith('#'): - folder, *owners = line.split() - folder_to_owners[folder] = owners - - for folder, owners in folder_to_owners.items(): - if folder == "*": - # skip the fallback here, we want to check all other patterns first - continue - - # Check if the pattern matches the relative path of the code owner - regex_pattern = '.*' + re.escape(code_owner) + '.*' - for owner in owners: - if re.match(regex_pattern, owner): - matched_folders.append(folder.strip('/')) - break - - return convert_iota_to_sui_folders(matched_folders) + for key in premap: + if key in str: + return str.replace(key, premap[key]) + + return str.replace('iota', 'sui') # Get the commits of a folder in the specified range def get_folder_commits(folder, start_ref, end_ref): @@ -105,18 +77,33 @@ def get_folder_commits(folder, start_ref, end_ref): if not is_commit_hash(end_ref): subprocess.run(["git", "fetch", "origin", "tag", end_ref], check=True) + if not os.path.exists(folder): + print(f"WARNING: Folder '{folder}' does not exist.") + return [] + # Define the git log command - git_log_command = ["git", "log", f"{start_ref}..{end_ref}", "--format=format:https://github.com/MystenLabs/sui/commit/%H", "--", folder] + git_log_command = ["git", "log", f"{start_ref}..{end_ref}", "--format=%H", "--", folder] + # Execute the git log command and collect the output result = subprocess.run(git_log_command, capture_output=True, text=True) git_log_output = result.stdout.strip().split('\n') if result.stdout.strip() else [] return git_log_output +# Run the git command to get the short commit message (title) +def get_commit_short_message(commit_hash): + short_message = subprocess.check_output( + ['git', 'show', '-s', '--format=%s', commit_hash], + stderr=subprocess.STDOUT, + text=True + ).strip() + return short_message + def analyze_folder_commits(start_ref, end_ref, folders): print(f"SINCE: {start_ref}") print(f"UNTIL: {end_ref}") print(f"FOLDERS: {', '.join(folders)}") + # Only insert non-empty lists into crates_commits folders_commits = {} for folder in folders: @@ -133,6 +120,9 @@ def analyze_folder_commits(start_ref, end_ref, folders): for commit in set(folders_commits[folder1]).intersection(folders_commits[folder2]) ) + def print_commit(commit_hash): + print(f"- https://github.com/MystenLabs/sui/commit/{commit_hash}: {get_commit_short_message(commit_hash)}") + # Remove duplicate commits from each folder for folder in folders_commits: folders_commits[folder] = [commit for commit in folders_commits[folder] if commit not in duplicate_commits] @@ -142,20 +132,19 @@ def analyze_folder_commits(start_ref, end_ref, folders): if commits: print(f"\n\n## {folder} ({len(commits)})") for commit in reversed(commits): - print(f"- {commit}") - + print_commit(commit) + # Print the duplicate commits print(f"\n\n## Cross-folder commits ({len(duplicate_commits)})") for commit in reversed(list(duplicate_commits)): - print(f"- {commit}") - + print_commit(commit) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Track upstream commits for specified folders.') parser.add_argument('--since', required=True, help='Start commit hash or the tag for git log (e.g., "bb778828e36d53a7d91a27e55109f2f45621badc", "mainnet-v1.32.2"), it is EXCLUDED from the results.') parser.add_argument('--until', required=True, help='End commit hash or the tag for git log (e.g., "3ada97c109cc7ae1b451cb384a1f2cfae49c8d3e", "mainnet-v1.36.2"), it is INCLUDED in the results.') + parser.add_argument('--codeowners', nargs='+', help='Code owners of the folders (e.g., "@iotaledger/node @iotaledger/consensus").') parser.add_argument('--folders', nargs='+', help='List of folders relative to the project root to track (e.g., "crates/iota-core crates/iota-node").') - parser.add_argument('--codeowner', help='code owner of the folders (e.g., "node")') parser.add_argument('--repo-url', default="git@github.com:MystenLabs/sui.git", help="The URL to the repository. Can also be a local folder.") parser.add_argument('--repo-tag', default=None, help="The tag to checkout in the repository.") parser.add_argument('--target-folder', default="result", help="The path to the target folder.") @@ -169,25 +158,31 @@ def analyze_folder_commits(start_ref, end_ref, folders): if args.clone_source and not args.repo_tag: parser.error("--repo-tag must be specified if --clone-source is true") - # Check if folders or code owner is specified - if not args.folders and not args.codeowner: - print("No crates or code owner specified.") + # Check if codeowners or folders is specified + if not args.codeowners and not args.folders: + print("No codeowners or folders specified.") exit(1) - folders = [] - if args.folders: - folders = convert_iota_to_sui_folders(args.folders) - - if args.codeowner: + code_owners = None + if args.codeowners: print("Parsing the CODEOWNERS file...") # Get crates of the code owner base_path = pathlib.Path("../../").absolute().resolve() - folders.extend(get_folders_for_code_owner(os.path.join(base_path, '.github', 'CODEOWNERS'), args.codeowner)) + + code_owners = CodeOwners( + file_path=os.path.join(base_path, '.github', 'CODEOWNERS'), + path_mapping_func=iota_to_sui_mapping_func, + pattern_mapping_func=iota_to_sui_mapping_func, + ) + + folders = set() + if args.folders: + folders.update(args.folders) if args.clone_source: - # remove the target folder if it exists + # Check if the target folder already exists if os.path.exists(target_folder): - shutil.rmtree(target_folder) + raise Exception(f"Target folder '{target_folder}' already exists, please remove it manually or run without --clone-source.") # Clone the repository clone_repo( @@ -199,5 +194,10 @@ def analyze_folder_commits(start_ref, end_ref, folders): # Change working directory to the target folder os.chdir(target_folder) + if code_owners: + # Get folders of the code owner + paths = code_owners.match_paths_for_owners(target_folder, args.codeowners) + folders.update(paths) + # Analyze the commits of the folders analyze_folder_commits(args.since, args.until, folders) \ No newline at end of file diff --git a/scripts/utils/codeowners.py b/scripts/utils/codeowners.py new file mode 100644 index 00000000000..5384b05c19d --- /dev/null +++ b/scripts/utils/codeowners.py @@ -0,0 +1,185 @@ +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 +import os, fnmatch, argparse + +# Parses the CODEOWNERS file and holds all possible codeowners +class CodeOwners(object): + def __init__(self, file_path, path_mapping_func=None, pattern_mapping_func=None): + self.path_mapping_func = path_mapping_func + + self.ignored_dirs_patterns=[ + '/.cargo/', + '/.config/', + '/.git/', + '/.pnpm_store/', + '/.vscode/', + '/chocolatey/', + '/target/', + ] + + # helper class + class PatternWithOwners(object): + def __init__(self, pattern, owners): + self.pattern = pattern + self.owners = owners + + self.default_owners = None + self.patterns = [] + with open(file_path, 'r') as file: + for line in file: + line = line.strip() + + # Skip empty lines and comments + if not line or line.startswith('#'): + continue + + glob_pattern, *owners = line.split() + + # If the pattern is a wildcard, set the default owners + if glob_pattern == "*": + self.default_owners = owners + continue + + if pattern_mapping_func: + glob_pattern = pattern_mapping_func(glob_pattern) + + # Insert the glob_pattern at the beginning of the list to prioritize it + # Last match wins in the CODEOWNERS file + self.patterns.insert(0, PatternWithOwners(glob_pattern, owners)) + + # matches a path against a pattern + # if the pattern starts with `/`, match it relative to root_path. + # otherwise, match it anywhere in the repository hierarchy. + def match_pattern(self, path, pattern): + if pattern.startswith("/"): + # this a path relative to the root folder + pattern = pattern.lstrip("/") + else: + # this is a pattern that can match anywhere in the hierarchy, so we "unroot" it + pattern = f"**/{pattern}" + + # remove trailing slashes + path = path.rstrip("/") + pattern = pattern.rstrip("/") + + return fnmatch.fnmatch(os.path.normpath(path), pattern) + + # returns the owners for a specific path or the default owners if + # no match is found and match_default is True, otherwise None + def match_owners_for_path(self, path, match_default=True): + if self.path_mapping_func: + path = self.path_mapping_func(path) + + for pattern_with_owners in self.patterns: + if self.match_pattern(path, pattern_with_owners.pattern): + return pattern_with_owners.owners + + if match_default and self.default_owners: + return self.default_owners + + return None + + # scans the folder structure for folders matching specific codeowners + # and returns the paths of the matched folders. + # Subfolders are not scanned further if a match is found. (except wildcard) + def match_paths_for_owners(self, root_folder, owners): + def get_relative_path_to_root_folder(path): + relative = os.path.relpath(path, root_folder) + if self.path_mapping_func: + relative = self.path_mapping_func(relative) + + return relative + + def is_ignored_dir(path): + return any(self.match_pattern(path, ignored_dir) for ignored_dir in self.ignored_dirs_patterns) + + matched_relative_paths_filtered = {} # the results for the specified owners + matched_relative_paths_all = {} # needed to filter unmatched directories in the second pass + unmatched_directories = [] # track directories that might need default owners + + # first Pass: traverse and match folders with specified owners + for root, dirs, _ in os.walk(root_folder): + relative_root = get_relative_path_to_root_folder(root) + if is_ignored_dir(relative_root): + dirs.clear() # don't walk into the directory if it should be ignored + continue + + for subfolder in list(dirs): + relative_subfolder = get_relative_path_to_root_folder(os.path.join(root, subfolder)) + if is_ignored_dir(relative_subfolder): + dirs.remove(subfolder) # don't walk into the directory if it should be ignored + continue + + # check if this subfolder matches any owner + matched_owners = self.match_owners_for_path(relative_subfolder, match_default=False) + if matched_owners: + # check if any of the matched owners is in the specified owners + if any(matched_owner in owners for matched_owner in matched_owners): + if relative_subfolder not in matched_relative_paths_all: + matched_relative_paths_all[relative_subfolder] = None + + if relative_subfolder not in matched_relative_paths_filtered: + matched_relative_paths_filtered[relative_subfolder] = [] + matched_relative_paths_filtered[relative_subfolder].extend(matched_owners) + + dirs.remove(subfolder) # stop traversing this directory, there was a direct match + else: + if relative_subfolder not in matched_relative_paths_all: + matched_relative_paths_all[relative_subfolder] = None + + # not owned by the specified owners, don't walk into the directory + dirs.remove(subfolder) + else: + # add to unmatched directories for second pass + unmatched_directories.append((root, subfolder)) + + # second Pass: handle directories that should get default owners + for root, subfolder in unmatched_directories: + relative_root = get_relative_path_to_root_folder(root) + if is_ignored_dir(relative_root): + continue + + relative_subfolder = get_relative_path_to_root_folder(os.path.join(root, subfolder)) + + # we need to check if any of the subdirectories have a match with a defined owner + has_matching_subdirectory = any( + matched_relative_path.startswith(relative_subfolder) and matched_relative_path != relative_subfolder for matched_relative_path in list(matched_relative_paths_all) + ) + + # we also need to check if the parent directory was already matched in the second pass + has_matching_parent = any( + relative_subfolder.startswith(matched_relative_path) and matched_relative_path != relative_subfolder for matched_relative_path in list(matched_relative_paths_all) + ) + + if has_matching_subdirectory or has_matching_parent: + continue + + # assign default owners if specified + default_owners = self.match_owners_for_path(relative_subfolder, match_default=True) + + # check if any of the matched owners is in the specified owners + if default_owners and any(owner in owners for owner in default_owners): + if relative_subfolder not in matched_relative_paths_all: + matched_relative_paths_all[relative_subfolder] = None + + if relative_subfolder not in matched_relative_paths_filtered: + matched_relative_paths_filtered[relative_subfolder] = [] + matched_relative_paths_filtered[relative_subfolder].extend(default_owners) + + # remove the leading slash in the keys + return matched_relative_paths_filtered + +################################################################################ +if __name__ == "__main__": + # Argument parser setup + parser = argparse.ArgumentParser(description="Tool to search for folders in the repository matching specific code owners.") + parser.add_argument('--target-folder', default="../../", help="The path to the target folder.") + parser.add_argument('--codeowners', default="@iotaledger/node @iotaledger/core-protocol", nargs='+', help='Code owners of the folders (e.g., "@iotaledger/node @iotaledger/consensus").') + args = parser.parse_args() + + code_owners = CodeOwners(os.path.join(args.target_folder, ".github/CODEOWNERS")) + paths_with_owners = code_owners.match_paths_for_owners(args.target_folder, args.codeowners) + + print(f"Found {len(paths_with_owners)} paths for code owners {args.codeowners}:") + for path, owners in paths_with_owners.items(): + print(f" {path}: {owners}") From 20a35bfe7c5ec5f7d85edcdddf12bc28d0165ac8 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 15:34:21 +0100 Subject: [PATCH 07/13] scripts: add script to compare based on code ownership --- .github/CODEOWNERS | 8 ++- scripts/compare/compare.py | 81 ++++++++++++++++++++++++++++ scripts/compare/run.sh | 31 +++++++++++ scripts/slipstream/run_compare.sh | 39 -------------- scripts/slipstream/run_slipstream.sh | 4 -- scripts/slipstream/slipstream.py | 24 --------- 6 files changed, 118 insertions(+), 69 deletions(-) create mode 100644 scripts/compare/compare.py create mode 100755 scripts/compare/run.sh delete mode 100755 scripts/slipstream/run_compare.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cb5faa2324d..9b9e7879801 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -126,13 +126,17 @@ vercel.json @iotaledger/tooling /.github/CODEOWNERS @luca-moser @lzpap @miker83z @alexsporn # Scripts -/scripts/dependency_graphs/ @muXxer /scripts/cargo_sort/ @muXxer -/scripts/generate_files/ @muXxer /scripts/codesearch/ @muXxer +/scripts/compare/ @muXxer +/scripts/dependency_graphs/ @muXxer +/scripts/generate_files/ @muXxer /scripts/slipstream/ @muXxer /scripts/tooling/ @iotaledger/tooling +/scripts/track_upstream_commits/ @muXxer +/scripts/utils/ @muXxer +# Other /kiosk/ @iotaledger/vm-language /nre/ @iotaledger/node diff --git a/scripts/compare/compare.py b/scripts/compare/compare.py new file mode 100644 index 00000000000..bc559a14c27 --- /dev/null +++ b/scripts/compare/compare.py @@ -0,0 +1,81 @@ +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 +import os, sys, subprocess, argparse, tempfile +sys.path.append('../utils') +from codeowners import CodeOwners + +# create symlinks in target for the matching folders of the code owners +def create_symlinks(source, target, code_owners, owners): + # check if the target directory already exists + if os.path.exists(target): + raise Exception(f"Target directory '{target}' already exists.") + + # get matching folders for the codeowners + matching_paths = code_owners.match_paths_for_owners(source, owners) + if not matching_paths: + raise Exception("No matching folders found for the code owners.") + + # create target directory + os.makedirs(target) + + # create symlinks for the matching folders + for path in matching_paths: + source_path = os.path.abspath(os.path.join(source, path)) + target_path = os.path.abspath(os.path.join(target, path)) + os.makedirs(os.path.dirname(target_path), exist_ok=True) + os.symlink(source_path, target_path, target_is_directory=True) + + print(f"Symlinks created in '{target}'.") + +# open tool for comparison +def run_compare_tool(compare_tool_binary, compare_tool_arguments, source_folder, target_folder): + print(f"Opening {compare_tool_binary} for comparison between {source_folder} and {target_folder}...") + + cmd = [compare_tool_binary] + if compare_tool_arguments: + cmd = cmd + compare_tool_arguments.split(" ") + cmd = cmd + [source_folder, target_folder] + + subprocess.run(cmd) + +################################################################################ +if __name__ == "__main__": + # Argument parser setup + parser = argparse.ArgumentParser(description="Repository comparison tool with optional code ownership filter.") + parser.add_argument('--source-folder', required=True, help="The path to the source folder for comparison.") + parser.add_argument('--target-folder', required=True, help="The path to the target folder for comparison.") + parser.add_argument('--codeowners', nargs='+', help='Optionally filter folders by code owners (e.g., "node consensus").') + parser.add_argument('--codeowners-file', default="", help="The path to the code owners file.") + parser.add_argument('--compare-tool-binary', default="meld", help="The binary to use for comparison.") + parser.add_argument('--compare-tool-arguments', default="", help="The arguments to use for comparison.") + args = parser.parse_args() + + # get the folder the script is in + script_folder = os.path.dirname(os.path.realpath(__file__)) + + # get the folder paths + source_folder = os.path.abspath(os.path.expanduser(args.source_folder)) + target_folder = os.path.abspath(os.path.expanduser(args.target_folder)) + + # check if we want to filter by code owners + if args.codeowners: + # load the code owners file + code_owners_file = os.path.join(script_folder, "..", "..", ".github", "CODEOWNERS") + if args.codeowners_file != "": + code_owners_file = os.path.abspath(os.path.expanduser(args.codeowners_file)) + + code_owners = CodeOwners(code_owners_file) + + results = os.path.abspath(os.path.join(script_folder, "results")) + os.makedirs(results, exist_ok=True) + + # Create temporary root folder for the target with random name suffix + with tempfile.TemporaryDirectory(prefix="source_", dir=results) as temp_dir_source: + create_symlinks(source_folder, os.path.join(temp_dir_source, "main"), code_owners, args.codeowners) + + with tempfile.TemporaryDirectory(prefix="target_", dir=results) as temp_dir_target: + create_symlinks(target_folder, os.path.join(temp_dir_target, "main"), code_owners, args.codeowners) + + run_compare_tool(args.compare_tool_binary, args.compare_tool_arguments, temp_dir_source, temp_dir_target) + else: + run_compare_tool(args.compare_tool_binary, args.compare_tool_arguments, source_folder, target_folder) diff --git a/scripts/compare/run.sh b/scripts/compare/run.sh new file mode 100755 index 00000000000..b67560d38cb --- /dev/null +++ b/scripts/compare/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# usage: compare.py [-h] +# --source-folder SOURCE_FOLDER +# --target-folder TARGET_FOLDER +# [--codeowners CODEOWNERS [CODEOWNERS ...]] +# [--codeowners-file CODEOWNERS_FILE] +# [--compare-tool-binary COMPARE_TOOL_BINARY] +# [--compare-tool-arguments COMPARE_TOOL_ARGUMENTS] +# +# Repository comparison tool with optional code ownership filter. +# +# options: +# -h, --help show this help message and exit +# --source-folder SOURCE_FOLDER +# The path to the source folder for comparison. +# --target-folder TARGET_FOLDER +# The path to the target folder for comparison. +# --codeowners CODEOWNERS [CODEOWNERS ...] +# Optionally filter folders by code owners (e.g., "node consensus"). +# --codeowners-file CODEOWNERS_FILE +# The path to the code owners file. +# --compare-tool-binary COMPARE_TOOL_BINARY +# The binary to use for comparison. +# --compare-tool-arguments COMPARE_TOOL_ARGUMENTS +# The arguments to use for comparison. +source ../utils/python_venv_wrapper.sh + +$PYTHON_CMD compare.py \ + --source-folder ../slipstream/results/mainnet-v1.32.2/main \ + --target-folder ../slipstream/results/mainnet-v1.40.3/main \ + --codeowners @iotaledger/node \ No newline at end of file diff --git a/scripts/slipstream/run_compare.sh b/scripts/slipstream/run_compare.sh deleted file mode 100755 index 784f137a834..00000000000 --- a/scripts/slipstream/run_compare.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# OPTIONS: -# --config FILE_PATH # The path to the configuration file. -# --verbose # Print verbose output. -# --repo-url REPO_URL_OR_LOCAL_FOLDER # The URL to the repository. Can also be a local folder. -# --repo-tag TAG # The tag to checkout in the repository. -# --version # The semantic version to filter overwrites/patches if not found in the repo-tag. -# --target-folder FOLDER # The path to the target folder. -# --main-repository-folder-name FOLDER # The name of the main repository folder (subfolder of target-folder). -# --target-branch BRANCH # The branch to create and checkout in the target folder. -# --commit-between-steps # Create a commit between each step. -# --panic-on-linter-errors # Panic on linter errors (typos, cargo fmt, dprint, pnpm lint, cargo clippy). -# --clone-source # Clone the upstream repository. -# --clone-history # Clone the complete history of the upstream repository. -# --create-branch # Create a new branch in the target folder. -# --delete # Delete files or folders based on the rules in the config. -# --apply-path-renames # Apply path renames based on the rules in the config. -# --apply-code-renames # Apply code renames based on the rules in the config. -# --copy-overwrites # Copy and overwrite files listed in the config. -# --apply-patches # Apply git patches from the patches folder. -# --run-fix-typos # Run script to fix typos. -# --run-cargo-fmt # Run cargo fmt. -# --run-dprint-fmt # Run dprint fmt. -# --run-pnpm-prettier-fix # Run pnpm prettier:fix. -# --run-pnpm-lint-fix # Run pnpm lint:fix. -# --run-shell-commands # Run shell commands listed in the config. -# --run-cargo-clippy # Run cargo clippy. -# --recompile-framework-packages # Recompile the framework system packages and bytecode snapshots. -# --compare-results # Open tool for comparison. -# --compare-source-folder FOLDER # The path to the source folder for comparison. -# --compare-tool-binary BINARY # The binary to use for comparison. -# --compare-tool-arguments ARGUMENTS # The arguments to use for comparison. -source python_venv_wrapper.sh - -$PYTHON_CMD slipstream.py \ - --config config_slipstream.json \ - --target-folder results/mainnet-v1.40.3/main \ - --compare-results \ - --compare-source-folder results/mainnet-v1.32.2/main \ No newline at end of file diff --git a/scripts/slipstream/run_slipstream.sh b/scripts/slipstream/run_slipstream.sh index 33a676e059c..8c34484eb8f 100755 --- a/scripts/slipstream/run_slipstream.sh +++ b/scripts/slipstream/run_slipstream.sh @@ -26,10 +26,6 @@ # --run-shell-commands # Run shell commands listed in the config. # --run-cargo-clippy # Run cargo clippy. # --recompile-framework-packages # Recompile the framework system packages and bytecode snapshots. -# --compare-results # Open tool for comparison. -# --compare-source-folder FOLDER # The path to the source folder for comparison. -# --compare-tool-binary BINARY # The binary to use for comparison. -# --compare-tool-arguments ARGUMENTS # The arguments to use for comparison. source ../utils/python_venv_wrapper.sh #REPO_TAG=mainnet-v1.29.2 diff --git a/scripts/slipstream/slipstream.py b/scripts/slipstream/slipstream.py index addde33fd5a..bd8638e2ad9 100644 --- a/scripts/slipstream/slipstream.py +++ b/scripts/slipstream/slipstream.py @@ -830,17 +830,6 @@ def commit_changes(commit_message): else: print(" No changes to commit.") -# Open tool for comparison -def run_compare_tool(compare_tool_binary, compare_tool_arguments, source_folder, target_folder): - print(f"Opening {compare_tool_binary} for comparison between {source_folder} and {target_folder}...") - - cmd = [compare_tool_binary] - if compare_tool_arguments: - cmd = cmd + compare_tool_arguments.split(" ") - cmd = cmd + [source_folder, target_folder] - - subprocess.run(cmd) - ################################################################################ if __name__ == "__main__": # Argument parser setup @@ -871,10 +860,6 @@ def run_compare_tool(compare_tool_binary, compare_tool_arguments, source_folder, parser.add_argument('--run-shell-commands', action='store_true', help="Run shell commands listed in the config.") parser.add_argument('--run-cargo-clippy', action='store_true', help="Run cargo clippy.") parser.add_argument('--recompile-framework-packages', action='store_true', help="Recompile the framework system packages and bytecode snapshots.") - parser.add_argument('--compare-results', action='store_true', help="Open tool for comparison.") - parser.add_argument('--compare-source-folder', help="The path to the source folder for comparison.") - parser.add_argument('--compare-tool-binary', default="meld", help="The binary to use for comparison.") - parser.add_argument('--compare-tool-arguments', default="", help="The arguments to use for comparison.") # get the folder the script is in script_folder = os.path.dirname(os.path.realpath(__file__)) @@ -906,11 +891,6 @@ def run_compare_tool(compare_tool_binary, compare_tool_arguments, source_folder, print(f"Version not found in tag: \"{args.repo_tag}\", please provide a valid \"--version\" argument.") exit(1) - # get current root folder - source_folder = os.path.abspath(os.path.join(os.getcwd(), "..", "..")) - if args.compare_source_folder: - source_folder = os.path.abspath(args.compare_source_folder) - # Load the configuration config = load_slipstream_config(args.config) @@ -1111,7 +1091,3 @@ def recompile_framework_packages_func(): # Execute the recompile framework packages function in the main folder execute_in_subfolders(target_folder, recompile_framework_packages_func, filter=[args.main_repository_folder_name]) - - if args.compare_results: - # Open tool for comparison - run_compare_tool(args.compare_tool_binary, args.compare_tool_arguments, source_folder, target_folder) From 3fcf043d612ccc167f15152345e0f056574c320e Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 16:10:47 +0100 Subject: [PATCH 08/13] scripts: move more common funcs into utils --- .../slipstream/{run_slipstream.sh => run.sh} | 0 scripts/slipstream/slipstream.py | 132 +----------------- .../track_upstream_commits.py | 61 +------- scripts/utils/git_utils.py | 122 ++++++++++++++++ scripts/utils/rust_utils.py | 20 +++ 5 files changed, 148 insertions(+), 187 deletions(-) rename scripts/slipstream/{run_slipstream.sh => run.sh} (100%) create mode 100644 scripts/utils/git_utils.py create mode 100644 scripts/utils/rust_utils.py diff --git a/scripts/slipstream/run_slipstream.sh b/scripts/slipstream/run.sh similarity index 100% rename from scripts/slipstream/run_slipstream.sh rename to scripts/slipstream/run.sh diff --git a/scripts/slipstream/slipstream.py b/scripts/slipstream/slipstream.py index bd8638e2ad9..bc24cb44032 100644 --- a/scripts/slipstream/slipstream.py +++ b/scripts/slipstream/slipstream.py @@ -1,4 +1,7 @@ -import os, re, json, shutil, subprocess, argparse, semver +import sys, os, re, json, shutil, subprocess, argparse, semver +sys.path.append('../utils') +from git_utils import clone_repo +from rust_utils import parse_rust_toolchain_version # Extract the semantic version from a version string def extract_sem_version(version_str): @@ -15,11 +18,6 @@ def load_slipstream_config(file_path): with open(file_path, 'r') as config_file: return json.load(config_file) -# Check if a reference is a commit hash -def is_commit_hash(ref): - # A commit hash is a 40-character hexadecimal string - return bool(re.match(r'^[0-9a-f]{40}$', ref)) - # Execute a function in all subfolders of the given folder (not recursive) def execute_in_subfolders(root_folder, function, filter=None): # Remember the current working directory @@ -42,111 +40,6 @@ def execute_in_subfolders(root_folder, function, filter=None): # Change working directory back to the current folder os.chdir(current_folder) -# Clone a repository (either from a URL or a local folder) -def clone_repo(repo_url, repo_tag, clone_history, target_folder, ignored_folders, ignored_files, ignored_file_types): - print(f"Cloning '{repo_url}' with tag '{repo_tag}' to '{target_folder}'...") - repo_url_exp = os.path.expanduser(repo_url) - - # Check if the repository is a git repository or a local folder - if os.path.exists(repo_url_exp): - # Fetch the latest changes - subprocess.run(["git", "fetch", "--all"], cwd=repo_url_exp, check=True) - - # Checkout the tag in the source folder - subprocess.run(["git", "checkout", repo_tag], cwd=repo_url_exp, check=True) - - # helper function to check if the current reference is following a branch - def is_following_branch(): - # Run the git command to check the current reference - result = subprocess.run( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], - cwd=repo_url_exp, - capture_output=True, - text=True, - check=True - ) - # If the result is 'HEAD', we're not following a branch (detached state or similar) - return result.stdout.strip() != "HEAD" - - # Pull the latest changes - if is_following_branch(): - print(" Pulling latest changes...") - subprocess.run(["git", "pull"], cwd=repo_url_exp, check=True) - - # Check if the local folder equals the target folder - if os.path.abspath(repo_url_exp) != os.path.abspath(target_folder): - - # Compile the regex patterns for ignored folders, files, and file types - ignored_folders = [re.compile(pattern) for pattern in ignored_folders] - ignored_files = [re.compile(pattern) for pattern in ignored_files] - ignored_file_types = [re.compile(pattern) for pattern in ignored_file_types] - - # helper function to filter ignored items - def filter_ignored_items(dir, contents): - dir = os.path.relpath(dir, os.path.abspath(repo_url_exp)) - - if any(pattern.search(dir) for pattern in ignored_folders): - print(f" Skipping directory (regex): {dir}") - return contents - - ignored = [] - - # Check if the item is a folder and matches any ignored folder pattern - for item_name in contents: - item_path = os.path.join(dir, item_name) - - if os.path.isdir(item_path): - if any(pattern.search(item_path) for pattern in ignored_folders): - print(f" Skipping directory (regex): {item_path}") - ignored.append(item_name) - continue - else: - # Ignore specified file types using regex patterns - if any(pattern.search(os.path.splitext(item_name)[1]) for pattern in ignored_file_types): - print(f" Skipping file (type regex): {item_path}") - ignored.append(item_name) - continue - - # Ignore specified files using regex patterns - if any(pattern.search(item_name) for pattern in ignored_files): - print(f" Skipping file (regex): {item_path}") - ignored.append(item_name) - continue - - return ignored - - # Copy the local folder to the target folder - shutil.copytree( - repo_url_exp, - target_folder, - ignore=filter_ignored_items, - symlinks=True - ) - else: - # Clone the repository, the tag can be used as the branch name directly to checkout a specific tag in one step - cmd = ["git", "clone"] - - # If the repo_tag is a commit hash, we need to checkout the commit hash after cloning - if not is_commit_hash(repo_tag): - if not clone_history: - cmd += ["--depth", "1"] - - cmd += ["--single-branch", "--branch", repo_tag, repo_url, target_folder] - - subprocess.run(cmd, check=True) - - # Change working directory to the cloned repo - os.chdir(target_folder) - else: - # Clone the repository without checking out the tag - subprocess.run(cmd + [repo_url, target_folder], check=True) - - # Change the working directory to the target folder - os.chdir(target_folder) - - # Checkout the tag in the target folder - subprocess.run(["git", "checkout", repo_tag], check=True) - # Search the Cargo.toml file for external crates and clone them def clone_external_crates(target_folder, config, main_repository_folder_name): print("Searching external crates to clone...") @@ -650,23 +543,6 @@ def run_dprint_fmt(panic_on_errors): print("Running dprint fmt...") subprocess.run(["dprint", "fmt"], check=panic_on_errors) -# Parse the rust-toolchain.toml file to get the Rust version -def parse_rust_toolchain_version(): - try: - content = None - with open('rust-toolchain.toml', 'r') as file: - content = file.read() - - # Regex to find the Rust version - match = re.search(r'(?<=channel = ").*(?=")', content) - if not match: - raise Exception("Rust version not found in rust-toolchain.toml.") - - return match.group(0) - - except FileNotFoundError: - raise FileNotFoundError("rust-toolchain.toml not found.") - # Prepare the Docker container for running turborepo def prepare_docker_turborepo(script_folder): rust_toolchain_version = parse_rust_toolchain_version() diff --git a/scripts/track_upstream_commits/track_upstream_commits.py b/scripts/track_upstream_commits/track_upstream_commits.py index cc63faf42d7..d1b9a9f6c60 100644 --- a/scripts/track_upstream_commits/track_upstream_commits.py +++ b/scripts/track_upstream_commits/track_upstream_commits.py @@ -3,56 +3,7 @@ import os, sys, subprocess, argparse, re, pathlib, shutil sys.path.append('../utils') from codeowners import CodeOwners - -def is_commit_hash(ref): - # A commit hash is a 40-character hexadecimal string - return bool(re.match(r'^[0-9a-f]{40}$', ref)) - -# Clone a repository (either from a URL or a local folder) -def clone_repo(repo_url, repo_tag, target_folder): - print(f"Cloning '{repo_url}' with tag '{repo_tag}' to '{target_folder}'...") - repo_url_exp = os.path.expanduser(repo_url) - - # Check if the repository is a git repository or a local folder - if os.path.exists(repo_url_exp): - # Fetch the latest changes - subprocess.run(["git", "fetch", "--all"], cwd=repo_url_exp, check=True) - - # Checkout the tag in the source folder - subprocess.run(["git", "checkout", repo_tag], cwd=repo_url_exp, check=True) - - # helper function to check if the current reference is following a branch - def is_following_branch(): - # Run the git command to check the current reference - result = subprocess.run( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], - cwd=repo_url_exp, - capture_output=True, - text=True, - check=True - ) - # If the result is 'HEAD', we're not following a branch (detached state or similar) - return result.stdout.strip() != "HEAD" - - # Pull the latest changes - if is_following_branch(): - print(" Pulling latest changes...") - subprocess.run(["git", "pull"], cwd=repo_url_exp, check=True) - - # Check if the local folder equals the target folder - if os.path.abspath(repo_url_exp) != os.path.abspath(target_folder): - # Copy the local folder to the target folder - shutil.copytree( - repo_url_exp, - target_folder, - symlinks=True - ) - else: - # Clone the repository, the tag can be used as the branch name directly to checkout a specific tag in one step - subprocess.run(["git", "clone", "--single-branch", "--branch", repo_tag, repo_url, target_folder], check=True) - - # Change working directory to the cloned repo - os.chdir(target_folder) +from git_utils import is_commit_hash, get_commit_short_message, clone_repo def iota_to_sui_mapping_func(str): premap = { @@ -90,15 +41,6 @@ def get_folder_commits(folder, start_ref, end_ref): return git_log_output -# Run the git command to get the short commit message (title) -def get_commit_short_message(commit_hash): - short_message = subprocess.check_output( - ['git', 'show', '-s', '--format=%s', commit_hash], - stderr=subprocess.STDOUT, - text=True - ).strip() - return short_message - def analyze_folder_commits(start_ref, end_ref, folders): print(f"SINCE: {start_ref}") print(f"UNTIL: {end_ref}") @@ -188,6 +130,7 @@ def print_commit(commit_hash): clone_repo( args.repo_url, args.repo_tag, + True, # clone history target_folder, ) else: diff --git a/scripts/utils/git_utils.py b/scripts/utils/git_utils.py new file mode 100644 index 00000000000..7fcbf9af04a --- /dev/null +++ b/scripts/utils/git_utils.py @@ -0,0 +1,122 @@ +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 +import os, re, subprocess, shutil + +# Check if a reference is a commit hash +def is_commit_hash(ref): + # A commit hash is a 40-character hexadecimal string + return bool(re.match(r'^[0-9a-f]{40}$', ref)) + +# Run the git command to get the short commit message (title) +def get_commit_short_message(commit_hash): + short_message = subprocess.check_output( + ['git', 'show', '-s', '--format=%s', commit_hash], + stderr=subprocess.STDOUT, + text=True + ).strip() + return short_message + +# Clone a repository (either from a URL or a local folder) +def clone_repo(repo_url, repo_tag, clone_history, target_folder, ignored_folders=[], ignored_files=[], ignored_file_types=[]): + print(f"Cloning '{repo_url}' with tag '{repo_tag}' to '{target_folder}'...") + repo_url_exp = os.path.expanduser(repo_url) + + # Check if the repository is a git repository or a local folder + if os.path.exists(repo_url_exp): + # Fetch the latest changes + subprocess.run(["git", "fetch", "--all"], cwd=repo_url_exp, check=True) + + # Checkout the tag in the source folder + subprocess.run(["git", "checkout", repo_tag], cwd=repo_url_exp, check=True) + + # helper function to check if the current reference is following a branch + def is_following_branch(): + # Run the git command to check the current reference + result = subprocess.run( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + cwd=repo_url_exp, + capture_output=True, + text=True, + check=True + ) + # If the result is 'HEAD', we're not following a branch (detached state or similar) + return result.stdout.strip() != "HEAD" + + # Pull the latest changes + if is_following_branch(): + print(" Pulling latest changes...") + subprocess.run(["git", "pull"], cwd=repo_url_exp, check=True) + + # Check if the local folder equals the target folder + if os.path.abspath(repo_url_exp) != os.path.abspath(target_folder): + + # Compile the regex patterns for ignored folders, files, and file types + ignored_folders = [re.compile(pattern) for pattern in ignored_folders] + ignored_files = [re.compile(pattern) for pattern in ignored_files] + ignored_file_types = [re.compile(pattern) for pattern in ignored_file_types] + + # helper function to filter ignored items + def filter_ignored_items(dir, contents): + dir = os.path.relpath(dir, os.path.abspath(repo_url_exp)) + + if any(pattern.search(dir) for pattern in ignored_folders): + print(f" Skipping directory (regex): {dir}") + return contents + + ignored = [] + + # Check if the item is a folder and matches any ignored folder pattern + for item_name in contents: + item_path = os.path.join(dir, item_name) + + if os.path.isdir(item_path): + if any(pattern.search(item_path) for pattern in ignored_folders): + print(f" Skipping directory (regex): {item_path}") + ignored.append(item_name) + continue + else: + # Ignore specified file types using regex patterns + if any(pattern.search(os.path.splitext(item_name)[1]) for pattern in ignored_file_types): + print(f" Skipping file (type regex): {item_path}") + ignored.append(item_name) + continue + + # Ignore specified files using regex patterns + if any(pattern.search(item_name) for pattern in ignored_files): + print(f" Skipping file (regex): {item_path}") + ignored.append(item_name) + continue + + return ignored + + # Copy the local folder to the target folder + shutil.copytree( + repo_url_exp, + target_folder, + ignore=filter_ignored_items, + symlinks=True + ) + else: + # Clone the repository, the tag can be used as the branch name directly to checkout a specific tag in one step + cmd = ["git", "clone"] + + # If the repo_tag is a commit hash, we need to checkout the commit hash after cloning + if not is_commit_hash(repo_tag): + if not clone_history: + cmd += ["--depth", "1"] + + cmd += ["--single-branch", "--branch", repo_tag, repo_url, target_folder] + + subprocess.run(cmd, check=True) + + # Change working directory to the cloned repo + os.chdir(target_folder) + else: + # Clone the repository without checking out the tag + subprocess.run(cmd + [repo_url, target_folder], check=True) + + # Change the working directory to the target folder + os.chdir(target_folder) + + # Checkout the tag in the target folder + subprocess.run(["git", "checkout", repo_tag], check=True) diff --git a/scripts/utils/rust_utils.py b/scripts/utils/rust_utils.py new file mode 100644 index 00000000000..3e68dcd21b6 --- /dev/null +++ b/scripts/utils/rust_utils.py @@ -0,0 +1,20 @@ +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 +import re + +# Parse the rust-toolchain.toml file to get the Rust version +def parse_rust_toolchain_version(): + try: + content = None + with open('rust-toolchain.toml', 'r') as file: + content = file.read() + + # Regex to find the Rust version + match = re.search(r'(?<=channel = ").*(?=")', content) + if not match: + raise Exception("Rust version not found in rust-toolchain.toml.") + + return match.group(0) + + except FileNotFoundError: + raise FileNotFoundError("rust-toolchain.toml not found.") From 03cdaed4e29416265fea3f694c5154943b20d079 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 16:53:00 +0100 Subject: [PATCH 09/13] scripts: add script to create patches for differing folders --- .github/CODEOWNERS | 1 + scripts/create_patches/.gitignore | 1 + scripts/create_patches/create_patches.py | 161 +++++++++++++++++++++++ scripts/create_patches/run.sh | 36 +++++ 4 files changed, 199 insertions(+) create mode 100644 scripts/create_patches/.gitignore create mode 100644 scripts/create_patches/create_patches.py create mode 100755 scripts/create_patches/run.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9b9e7879801..fbd9d069692 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -129,6 +129,7 @@ vercel.json @iotaledger/tooling /scripts/cargo_sort/ @muXxer /scripts/codesearch/ @muXxer /scripts/compare/ @muXxer +/scripts/create_patches/ @muXxer /scripts/dependency_graphs/ @muXxer /scripts/generate_files/ @muXxer /scripts/slipstream/ @muXxer diff --git a/scripts/create_patches/.gitignore b/scripts/create_patches/.gitignore new file mode 100644 index 00000000000..f518c74db01 --- /dev/null +++ b/scripts/create_patches/.gitignore @@ -0,0 +1 @@ +/patches/ \ No newline at end of file diff --git a/scripts/create_patches/create_patches.py b/scripts/create_patches/create_patches.py new file mode 100644 index 00000000000..3aeac16dccd --- /dev/null +++ b/scripts/create_patches/create_patches.py @@ -0,0 +1,161 @@ +import sys, os, difflib, shutil, argparse +sys.path.append('../utils') +from codeowners import CodeOwners + +# Create a patch file for the differences between source_file and target_file. +def create_patch(source_file, target_file, patch_file, relative_item, verbose): + source_lines = [] + target_lines = [] + + from_file = "/dev/null" + to_file = "/dev/null" + + if os.path.exists(source_file): + from_file = relative_item + with open(source_file, 'r') as f: + try: + source_lines = f.readlines() + except UnicodeDecodeError as e: + if verbose: + print(f"Error: Unable to read source file {source_file}, {e}") + return + + if os.path.exists(target_file): + to_file = relative_item + with open(target_file, 'r') as f: + try: + target_lines = f.readlines() + except UnicodeDecodeError as e: + if verbose: + print(f"Error: Unable to read target file {target_file}, {e}") + return + + diff = difflib.unified_diff( + source_lines, + target_lines, + fromfile=from_file, + tofile=to_file + ) + + file_diff = ''.join(diff) + if len(file_diff) == 0: + # No differences found + return + + # Create output folder if it doesn't exist + if not os.path.exists(os.path.dirname(patch_file)): + os.makedirs(os.path.dirname(patch_file)) + + with open(patch_file, 'w') as patch: + patch.write(file_diff) + +# Recursively process folders and generate patch files for differing files. +def process_folder(source_folder_root, target_folder_root, output_folder_root, relative_folder, verbose): + # Join relative folder path with folder paths + source_folder = os.path.join(source_folder_root, relative_folder) + target_folder = os.path.join(target_folder_root, relative_folder) + output_folder = os.path.join(output_folder_root, relative_folder) + + if verbose: + print(f"Processing folder: {output_folder}") + + # Get lists of files and subdirectories in both source and target + source_items = set(os.listdir(source_folder)) if os.path.exists(source_folder) else set() + target_items = set(os.listdir(target_folder)) if os.path.exists(target_folder) else set() + + # Combine source and target items + all_items = source_items.union(target_items) + + for item in all_items: + source_item = os.path.join(source_folder, item) + target_item = os.path.join(target_folder, item) + relative_item = os.path.join(relative_folder, item) + + if not os.path.exists(source_item) and not os.path.exists(target_item): + raise Exception(f"Error: {os.path.join(relative_folder, item)} does not exist in either source or target.") + + if os.path.isdir(source_item) or os.path.isdir(target_item): + # Recursively process subdirectories + process_folder(source_folder_root, target_folder_root, output_folder_root, relative_item, verbose) + else: + # Process files + patch_file = os.path.join(output_folder, f"{item}.patch") + if verbose: + print(f"Comparing: {item}") + create_patch(source_item, target_item, patch_file, relative_item, verbose) + if verbose: + print(f"Patch created: {patch_file}") + +# Process specified folders and generate patch files for differing files. +def process_folders(source_folder, target_folder, folders, output_folder, verbose): + # remove the output folder if it exists + if os.path.exists(output_folder): + print(f"Removing existing output folder: {output_folder}") + shutil.rmtree(output_folder) + + # create the output folder + os.makedirs(output_folder) + + for folder in folders: + relative_folder = os.path.normpath(folder) + process_folder(source_folder, target_folder, output_folder, relative_folder, verbose) + +################################################################################ +if __name__ == "__main__": + # Argument parser setup + parser = argparse.ArgumentParser(description="Generate patch files for differing files in specified subfolders.") + parser.add_argument('--source-folder', required=True, help="The path to the source folder for comparison.") + parser.add_argument('--target-folder', required=True, help="The path to the target folder for comparison.") + parser.add_argument("--output-folder", required=True, help="The path to the output folder to save patch files.") + parser.add_argument('--folders', nargs='+', help='Optionally filter by a list of folders relative to the project root (e.g., "crates/iota-core crates/iota-node").') + parser.add_argument('--codeowners', nargs='+', help='Optionally filter folders by code owners (e.g., "@iotaledger/node @iotaledger/consensus").') + parser.add_argument('--codeowners-file', default="", help="The path to the code owners file.") + parser.add_argument('--verbose', action='store_true', help="Print verbose output.") + args = parser.parse_args() + + # get the source folder + source_folder = args.source_folder + if not source_folder: + raise Exception("Error: No source folder specified.") + source_folder = os.path.abspath(os.path.expanduser(source_folder)) + if not os.path.exists(source_folder): + raise Exception(f"Error: Source folder {source_folder} does not exist.") + + # get the target folder + target_folder = args.target_folder + if not target_folder: + raise Exception("Error: No target folder specified.") + target_folder = os.path.abspath(os.path.expanduser(target_folder)) + if not os.path.exists(target_folder): + raise Exception(f"Error: Target folder {target_folder} does not exist.") + + # get the output folder + output_folder = args.output_folder + if not output_folder: + raise Exception("Error: No output folder specified.") + + output_folder = os.path.abspath(os.path.expanduser(output_folder)) + + folders = set() + if args.folders: + folders.update(args.folders) + + # check if we want to filter by code owners + if args.codeowners: + # get the folder the script is in + script_folder = os.path.dirname(os.path.realpath(__file__)) + + # load the code owners file + code_owners_file = os.path.join(script_folder, "..", "..", ".github", "CODEOWNERS") + if args.codeowners_file != "": + code_owners_file = os.path.abspath(os.path.expanduser(args.codeowners_file)) + + code_owners = CodeOwners(code_owners_file) + + # Get folders of the code owner + paths = code_owners.match_paths_for_owners(source_folder, args.codeowners) + folders.update(paths) + paths = code_owners.match_paths_for_owners(target_folder, args.codeowners) + folders.update(paths) + + process_folders(source_folder, target_folder, folders, output_folder, args.verbose) diff --git a/scripts/create_patches/run.sh b/scripts/create_patches/run.sh new file mode 100755 index 00000000000..012ca8e679c --- /dev/null +++ b/scripts/create_patches/run.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# usage: create_patches.py [-h] +# --source-folder SOURCE_FOLDER +# --target-folder TARGET_FOLDER +# --output-folder OUTPUT_FOLDER +# [--folders FOLDERS [FOLDERS ...]] +# [--codeowners CODEOWNERS [CODEOWNERS ...]] +# [--codeowners-file CODEOWNERS_FILE] +# [--verbose] +# +# Generate patch files for differing files in specified subfolders. +# +# options: +# -h, --help show this help message and exit +# --source-folder SOURCE_FOLDER +# The path to the source folder for comparison. +# --target-folder TARGET_FOLDER +# The path to the target folder for comparison. +# --folders FOLDERS [FOLDERS ...] +# Optionally filter by a list of folders relative to the project root (e.g., "crates/iota-core crates/iota-node"). +# --codeowners CODEOWNERS [CODEOWNERS ...] +# Optionally filter folders by code owners (e.g., "@iotaledger/node @iotaledger/consensus"). +# --codeowners-file CODEOWNERS_FILE +# The path to the code owners file. +# --output-folder OUTPUT_FOLDER +# The path to the output folder to save patch files. +# --verbose Print verbose output. +source ../utils/python_venv_wrapper.sh + +$PYTHON_CMD create_patches.py \ + --source-folder ../slipstream/results/mainnet-v1.32.2/main \ + --target-folder ../slipstream/results/mainnet-v1.41.1/main \ + --output-folder patches \ + --codeowners @iotaledger/node \ + --verbose \ + "$@" \ No newline at end of file From 559d5676e8c9b1cfc052975733e27fc06e9645d2 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 16:53:28 +0100 Subject: [PATCH 10/13] scripts: improve gitignore files --- .gitignore | 1 - scripts/compare/.gitignore | 1 + scripts/dependency_graphs/.gitignore | 1 + scripts/slipstream/.gitignore | 1 + 4 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 scripts/compare/.gitignore create mode 100644 scripts/dependency_graphs/.gitignore create mode 100644 scripts/slipstream/.gitignore diff --git a/.gitignore b/.gitignore index 7013bcbc7ce..f1449a8c081 100644 --- a/.gitignore +++ b/.gitignore @@ -72,7 +72,6 @@ yarn-error.log* .env checkpoints_dir/* light_client.yaml -scripts/dependency_graphs/output/* lcov.info **/venv/ diff --git a/scripts/compare/.gitignore b/scripts/compare/.gitignore new file mode 100644 index 00000000000..8b22acb4306 --- /dev/null +++ b/scripts/compare/.gitignore @@ -0,0 +1 @@ +/results/ \ No newline at end of file diff --git a/scripts/dependency_graphs/.gitignore b/scripts/dependency_graphs/.gitignore new file mode 100644 index 00000000000..2c04c9d8c77 --- /dev/null +++ b/scripts/dependency_graphs/.gitignore @@ -0,0 +1 @@ +/output/ \ No newline at end of file diff --git a/scripts/slipstream/.gitignore b/scripts/slipstream/.gitignore new file mode 100644 index 00000000000..8b22acb4306 --- /dev/null +++ b/scripts/slipstream/.gitignore @@ -0,0 +1 @@ +/results/ \ No newline at end of file From 523dff28b1dcd043d1b6ce8dcae313d12bb190d7 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 16:53:58 +0100 Subject: [PATCH 11/13] scripts: unify run scrips --- scripts/compare/compare.py | 8 +++--- scripts/compare/run.sh | 7 +++-- scripts/track_upstream_commits/run.sh | 28 ++++++++++++++----- .../track_upstream_commits.py | 19 ++++++++----- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/scripts/compare/compare.py b/scripts/compare/compare.py index bc559a14c27..b6847f86db8 100644 --- a/scripts/compare/compare.py +++ b/scripts/compare/compare.py @@ -44,21 +44,21 @@ def run_compare_tool(compare_tool_binary, compare_tool_arguments, source_folder, parser = argparse.ArgumentParser(description="Repository comparison tool with optional code ownership filter.") parser.add_argument('--source-folder', required=True, help="The path to the source folder for comparison.") parser.add_argument('--target-folder', required=True, help="The path to the target folder for comparison.") - parser.add_argument('--codeowners', nargs='+', help='Optionally filter folders by code owners (e.g., "node consensus").') + parser.add_argument('--codeowners', nargs='+', help='Optionally filter folders by code owners (e.g., "@iotaledger/node @iotaledger/consensus").') parser.add_argument('--codeowners-file', default="", help="The path to the code owners file.") parser.add_argument('--compare-tool-binary', default="meld", help="The binary to use for comparison.") parser.add_argument('--compare-tool-arguments', default="", help="The arguments to use for comparison.") args = parser.parse_args() - # get the folder the script is in - script_folder = os.path.dirname(os.path.realpath(__file__)) - # get the folder paths source_folder = os.path.abspath(os.path.expanduser(args.source_folder)) target_folder = os.path.abspath(os.path.expanduser(args.target_folder)) # check if we want to filter by code owners if args.codeowners: + # get the folder the script is in + script_folder = os.path.dirname(os.path.realpath(__file__)) + # load the code owners file code_owners_file = os.path.join(script_folder, "..", "..", ".github", "CODEOWNERS") if args.codeowners_file != "": diff --git a/scripts/compare/run.sh b/scripts/compare/run.sh index b67560d38cb..8b7e64e88fd 100755 --- a/scripts/compare/run.sh +++ b/scripts/compare/run.sh @@ -16,7 +16,7 @@ # --target-folder TARGET_FOLDER # The path to the target folder for comparison. # --codeowners CODEOWNERS [CODEOWNERS ...] -# Optionally filter folders by code owners (e.g., "node consensus"). +# Optionally filter folders by code owners (e.g., "@iotaledger/node @iotaledger/consensus"). # --codeowners-file CODEOWNERS_FILE # The path to the code owners file. # --compare-tool-binary COMPARE_TOOL_BINARY @@ -27,5 +27,6 @@ source ../utils/python_venv_wrapper.sh $PYTHON_CMD compare.py \ --source-folder ../slipstream/results/mainnet-v1.32.2/main \ - --target-folder ../slipstream/results/mainnet-v1.40.3/main \ - --codeowners @iotaledger/node \ No newline at end of file + --target-folder ../slipstream/results/mainnet-v1.41.1/main \ + --codeowners @iotaledger/node \ + "$@" \ No newline at end of file diff --git a/scripts/track_upstream_commits/run.sh b/scripts/track_upstream_commits/run.sh index ecdd41ebd5b..70e0146f2fb 100755 --- a/scripts/track_upstream_commits/run.sh +++ b/scripts/track_upstream_commits/run.sh @@ -1,13 +1,27 @@ #!/bin/bash -# OPTIONS: +# usage: track_upstream_commits.py [-h] +# --since SINCE +# --until UNTIL +# [--codeowners CODEOWNERS [CODEOWNERS ...]] +# [--codeowners-file CODEOWNERS_FILE] +# [--folders FOLDERS [FOLDERS ...]] +# [--repo-url REPO_URL] +# [--repo-tag REPO_TAG] +# [--target-folder TARGET_FOLDER] +# [--clone-source] +# +# Track upstream commits for specified folders. +# +# options: # -h, --help show this help message and exit -# --since SINCE Start commit hash or the tag for git log (e.g., "bb778828e36d53a7d91a27e55109f2f45621badc", "mainnet-v1.32.2"), it is EXCLUDED from the -# results. +# --since SINCE Start commit hash or the tag for git log (e.g., "bb778828e36d53a7d91a27e55109f2f45621badc", "mainnet-v1.32.2"), it is EXCLUDED from the results. # --until UNTIL End commit hash or the tag for git log (e.g., "3ada97c109cc7ae1b451cb384a1f2cfae49c8d3e", "mainnet-v1.36.2"), it is INCLUDED in the results. +# --codeowners CODEOWNERS [CODEOWNERS ...] +# Code owners of the folders (e.g., "@iotaledger/node @iotaledger/consensus"). +# --codeowners-file CODEOWNERS_FILE +# The path to the code owners file. # --folders FOLDERS [FOLDERS ...] # List of folders relative to the project root to track (e.g., "crates/iota-core crates/iota-node"). -# --codeowner CODEOWNER -# code owner of the folders (e.g., "node") # --repo-url REPO_URL The URL to the repository. Can also be a local folder. # --repo-tag REPO_TAG The tag to checkout in the repository. # --target-folder TARGET_FOLDER @@ -16,7 +30,7 @@ source ../utils/python_venv_wrapper.sh $PYTHON_CMD track_upstream_commits.py \ - --repo-tag "mainnet-v1.36.2" \ + --repo-tag "mainnet-v1.41.1" \ --target-folder result \ --clone-source \ - "$@" \ No newline at end of file + "$@" diff --git a/scripts/track_upstream_commits/track_upstream_commits.py b/scripts/track_upstream_commits/track_upstream_commits.py index d1b9a9f6c60..465079f3a8c 100644 --- a/scripts/track_upstream_commits/track_upstream_commits.py +++ b/scripts/track_upstream_commits/track_upstream_commits.py @@ -86,6 +86,7 @@ def print_commit(commit_hash): parser.add_argument('--since', required=True, help='Start commit hash or the tag for git log (e.g., "bb778828e36d53a7d91a27e55109f2f45621badc", "mainnet-v1.32.2"), it is EXCLUDED from the results.') parser.add_argument('--until', required=True, help='End commit hash or the tag for git log (e.g., "3ada97c109cc7ae1b451cb384a1f2cfae49c8d3e", "mainnet-v1.36.2"), it is INCLUDED in the results.') parser.add_argument('--codeowners', nargs='+', help='Code owners of the folders (e.g., "@iotaledger/node @iotaledger/consensus").') + parser.add_argument('--codeowners-file', default="", help="The path to the code owners file.") parser.add_argument('--folders', nargs='+', help='List of folders relative to the project root to track (e.g., "crates/iota-core crates/iota-node").') parser.add_argument('--repo-url', default="git@github.com:MystenLabs/sui.git", help="The URL to the repository. Can also be a local folder.") parser.add_argument('--repo-tag', default=None, help="The tag to checkout in the repository.") @@ -107,15 +108,19 @@ def print_commit(commit_hash): code_owners = None if args.codeowners: - print("Parsing the CODEOWNERS file...") - # Get crates of the code owner - base_path = pathlib.Path("../../").absolute().resolve() + # get the folder the script is in + script_folder = os.path.dirname(os.path.realpath(__file__)) + + # load the code owners file + code_owners_file = os.path.join(script_folder, "..", "..", ".github", "CODEOWNERS") + if args.codeowners_file != "": + code_owners_file = os.path.abspath(os.path.expanduser(args.codeowners_file)) code_owners = CodeOwners( - file_path=os.path.join(base_path, '.github', 'CODEOWNERS'), - path_mapping_func=iota_to_sui_mapping_func, - pattern_mapping_func=iota_to_sui_mapping_func, - ) + file_path=code_owners_file, + path_mapping_func=iota_to_sui_mapping_func, + pattern_mapping_func=iota_to_sui_mapping_func, + ) folders = set() if args.folders: From fa21a6396b4d43bbecace4b3e4668c965dbc33d8 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 17:02:04 +0100 Subject: [PATCH 12/13] scripts: re-add ignore for `node_modules` in `search_tests.py` --- scripts/codesearch/search_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/codesearch/search_tests.py b/scripts/codesearch/search_tests.py index fd7e3a105ea..168bc4d36a1 100644 --- a/scripts/codesearch/search_tests.py +++ b/scripts/codesearch/search_tests.py @@ -120,6 +120,7 @@ def merge_results_func(results, result): 'linting', 'scripts', 'target', + 'node_modules', ], file_pattern=re.compile(r'\.rs$', flags=re.IGNORECASE), search_in_file_func=search_in_file_func, From c6c8426aad157e10e29af2f1985a76ee17f05b94 Mon Sep 17 00:00:00 2001 From: muXxer Date: Fri, 31 Jan 2025 17:05:37 +0100 Subject: [PATCH 13/13] scripts: add missing licenses --- scripts/cargo_sort/cargo_sort.py | 2 ++ scripts/codesearch/codesearch.py | 2 ++ scripts/codesearch/search_tests.py | 4 +++- scripts/codesearch/search_versions.py | 2 ++ scripts/create_patches/create_patches.py | 2 ++ scripts/slipstream/slipstream.py | 2 ++ 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/cargo_sort/cargo_sort.py b/scripts/cargo_sort/cargo_sort.py index af358a3a769..590e275994c 100644 --- a/scripts/cargo_sort/cargo_sort.py +++ b/scripts/cargo_sort/cargo_sort.py @@ -1,3 +1,5 @@ +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 import os, re, argparse, subprocess COMMENT_DEPENDENCIES_START_EXTERNAL = "# external dependencies" diff --git a/scripts/codesearch/codesearch.py b/scripts/codesearch/codesearch.py index f803d03920d..aa7535f1f7d 100644 --- a/scripts/codesearch/codesearch.py +++ b/scripts/codesearch/codesearch.py @@ -1,3 +1,5 @@ +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 import os, re from tqdm import tqdm from concurrent.futures import ProcessPoolExecutor, as_completed diff --git a/scripts/codesearch/search_tests.py b/scripts/codesearch/search_tests.py index 168bc4d36a1..8e0f0a501bd 100644 --- a/scripts/codesearch/search_tests.py +++ b/scripts/codesearch/search_tests.py @@ -1,4 +1,6 @@ -import re, argparse, time +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 +import re, argparse from collections import defaultdict # import all funcs from codesearch.py diff --git a/scripts/codesearch/search_versions.py b/scripts/codesearch/search_versions.py index 8a3c20db243..8906b55e982 100644 --- a/scripts/codesearch/search_versions.py +++ b/scripts/codesearch/search_versions.py @@ -1,3 +1,5 @@ +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 import os, re, argparse from collections import defaultdict diff --git a/scripts/create_patches/create_patches.py b/scripts/create_patches/create_patches.py index 3aeac16dccd..6977ba6b9ca 100644 --- a/scripts/create_patches/create_patches.py +++ b/scripts/create_patches/create_patches.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 import sys, os, difflib, shutil, argparse sys.path.append('../utils') from codeowners import CodeOwners diff --git a/scripts/slipstream/slipstream.py b/scripts/slipstream/slipstream.py index bc24cb44032..6dea29e8a50 100644 --- a/scripts/slipstream/slipstream.py +++ b/scripts/slipstream/slipstream.py @@ -1,3 +1,5 @@ +# Copyright (c) 2024 IOTA Stiftung +# SPDX-License-Identifier: Apache-2.0 import sys, os, re, json, shutil, subprocess, argparse, semver sys.path.append('../utils') from git_utils import clone_repo