diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml new file mode 100644 index 0000000000000..949fd9940f0d7 --- /dev/null +++ b/.github/workflows/sync_native_dependency.yml @@ -0,0 +1,22 @@ +name: Compare Dependency Constraints +on: + pull_request: + branches: + - master +jobs: + compare_dependency_constraints_script: + runs-on: ubuntu-latest + if: github.repository == 'demisto/dockerFiles' + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.*" + - name: Install dependencies with pipenv + run: | + python -m pip install --upgrade pip + pip install pipenv + pipenv install + - name: Compare dependency constraints + run: pipenv run python utils/compare_dependency_constraints.py \ No newline at end of file diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 3e43b2b7946f8..b4b6afc9d453c 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -3,101 +3,226 @@ from typing import Any, NamedTuple import requests import toml +import sys DOCKER_FOLDER = Path(__file__).parent.parent / "docker" NATIVE_IMAGE = "py3-native" +PY3_TOOLS_UBI_IMAGE = "py3-tools-ubi" +PY3_TOOLS_IMAGE = "py3-tools" +PYPROJECT = "pyproject.toml" +PIPFILE = "Pipfile" -def parse_constraints(dir_name: str) -> dict[str, str]: +class Discrepancy(NamedTuple): + """Represents a discrepancy between dependencies in different images.""" + + dependency: str + image: str + reference_image: str + path: Path + in_image: str | None = None + in_reference: str | None = None + + def __str__(self) -> str: + return ( + f"{self.dependency} is {self.in_image or 'missing'} in {self.image}, " + f"but {self.in_reference or 'missing'} in the {self.reference_image} image. " + "This discrepancy may cause issues when running content." + ) + + +def get_dependency_file_path(dir_name: str) -> Path: + """Returns the path to the dependency file (Pipfile or pyproject.toml) in the given directory.""" dir_path = DOCKER_FOLDER / dir_name + if not dir_path.exists(): - raise FileNotFoundError(dir_path) + raise FileNotFoundError(f"Directory {dir_path} does not exist.") - pip_path = dir_path / "Pipfile" - pyproject_path = dir_path / "pyproject.toml" + pip_path = dir_path / PIPFILE + pyproject_path = dir_path / PYPROJECT if pip_path.exists() and pyproject_path.exists(): raise ValueError( - f"Can't have both pyproject and Pipfile in a dockerfile folder ({dir_name})" + f"Can't have both pyproject and Pipfile in a dockerfile folder ({dir_path})" ) - if pip_path.exists(): - return lower_dict_keys(_parse_pipfile(pip_path)) + return pip_path if pyproject_path.exists(): - return lower_dict_keys(_parse_pyproject(pyproject_path)) + return pyproject_path + + raise ValueError(f"Neither pyproject nor Pipfile found in {dir_path}") - raise ValueError(f"Neither pyproject nor Pipfile found in {dir_name}") + +def parse_constraints(name: str) -> dict[str, str]: + """Parses the dependency constraints from the given image name.""" + path = get_dependency_file_path(name) + if path.suffix == PIPFILE: + return lower_dict_keys(_parse_pipfile(path)) + + return lower_dict_keys(_parse_pyproject(path)) def _parse_pipfile(path: Path) -> dict[str, str]: + """Parses the Pipfile and returns the dependencies.""" return toml.load(path).get("packages", {}) def _parse_pyproject(path: Path) -> dict[str, str]: + """Parses the pyproject.toml file and returns the dependencies.""" return toml.load(path).get("tool", {}).get("poetry", {}).get("dependencies", {}) def lower_dict_keys(dictionary: dict[str, Any]) -> dict[str, Any]: + """Converts all keys in the dictionary to lowercase.""" return {k.lower(): v for k, v in dictionary.items()} -class Discrepancy(NamedTuple): - dependency: str - image: str - in_image: str | None = None - in_native: str | None = None +def find_library_line_number(lib_name: str, file_path: Path) -> int: + """ + Searches for a library in the pyproject.toml or Pipfile file and returns the line number where it is found. - def __str__(self) -> str: - return f"{self.dependency}: {self.in_image or 'missing'} in {self.image}, {self.in_native or 'missing'} in native" + Parameters: + - lib_name: The name of the library to search for. + - file_path: The directory containing the pyproject.toml or Pipfile. + + Returns: + - The line number containing the library name, or 1 if the library is not found. + """ + for line_number, line in enumerate( + file_path.read_text().splitlines(), start=1 + ): # Start counting from line 1 + if lib_name in line: + return line_number + + return 1 # default + + +def compare_constraints(images_contained_in_native: list[str]) -> int: + """Compares the dependency constraints between different images and reports discrepancies. + + This function compares the dependencies of the following images: + - `py3-tools` + - `py3-tools-ubi` + - `native` + + against the dependencies of the images listed in `images_contained_in_native`. + Additionally, it compares the dependencies of `py3-tools` against `py3-tools-ubi`. + + Args: + images_contained_in_native (list[str]): A list of image names to compare against the native image. + + Returns: + int: Returns 1 if there are discrepancies, 0 otherwise. + """ -def compare_constraints(images_contained_in_native: list[str]): native_constraints = ( - parse_constraints("python3-ubi") - | parse_constraints("py3-tools-ubi") + parse_constraints(PY3_TOOLS_IMAGE) + | parse_constraints(PY3_TOOLS_UBI_IMAGE) | parse_constraints(NATIVE_IMAGE) ) - native_constraint_keys = set(native_constraints.keys()) + py3_tools_constraints = parse_constraints(PY3_TOOLS_IMAGE) + py3_tools_ubi_constraints = parse_constraints(PY3_TOOLS_UBI_IMAGE) + discrepancies: list[Discrepancy] = [] for image in images_contained_in_native: - discrepancies: list[Discrepancy] = [] - - constraints = parse_constraints(image) - constraint_keys = set(constraints.keys()) - - discrepancies.extend( # image dependencies missing from native - ( - Discrepancy( - dependency=dependency, - image=image, - in_image=constraints[dependency], - ) - for dependency in sorted( - constraint_keys.difference(native_constraint_keys) - ) + discrepancies.extend(compare_with_native(image, native_constraints)) + + discrepancies.extend( + compare_py3_tools_with_ubi(py3_tools_constraints, py3_tools_ubi_constraints) + ) + + for discrepancy in discrepancies: + line_number = find_library_line_number(discrepancy.dependency, discrepancy.path) + print( + f"::error file={discrepancy.path},line={line_number},endLine={line_number},title=Native Image Discrepancy::{discrepancy}" + ) + return int(bool(discrepancies)) + + +def compare_with_native(image: str, native_constraints: dict) -> list[Discrepancy]: + path = get_dependency_file_path(image) + constraints = parse_constraints(image) + constraint_keys = set(constraints.keys()) + native_constraint_keys = set(native_constraints.keys()) + + discrepancies: list[Discrepancy] = [] + + discrepancies.extend( # image dependencies missing from native + ( + Discrepancy( + dependency=dependency, + image=image, + reference_image=NATIVE_IMAGE, + in_image=constraints[dependency], + path=path, + ) + for dependency in sorted(constraint_keys.difference(native_constraint_keys)) + ) + ) + discrepancies.extend( # shared dependencies with native, different versions + ( + Discrepancy( + dependency=dependency, + image=image, + reference_image=NATIVE_IMAGE, + in_image=constraints[dependency], + in_reference=native_constraints[dependency], + path=path, ) + for dependency in sorted( + constraint_keys.intersection(native_constraint_keys) + ) + if constraints[dependency] != native_constraints[dependency] ) - discrepancies.extend( # shared dependencies with native, different versions - ( - Discrepancy( - dependency=dependency, - image=image, - in_image=constraints[dependency], - in_native=native_constraints[dependency], - ) - for dependency in sorted( - constraint_keys.intersection(native_constraint_keys) - ) - if constraints[dependency] != native_constraints[dependency] + ) + + return discrepancies + + +def compare_py3_tools_with_ubi( + py3_tools_constraints: dict, py3_tools_ubi_constraints: dict +) -> list[Discrepancy]: + py3_tools_keys = set(py3_tools_constraints.keys()) + py3_tools_ubi_keys = set(py3_tools_ubi_constraints.keys()) + + discrepancies: list[Discrepancy] = [] + + discrepancies.extend( # py3-tools-ubi dependencies missing from py3-tools + ( + Discrepancy( + dependency=dependency, + image=PY3_TOOLS_UBI_IMAGE, + reference_image=PY3_TOOLS_IMAGE, + in_image=py3_tools_ubi_constraints.get(dependency), + in_reference=py3_tools_constraints.get(dependency), + path=get_dependency_file_path(PY3_TOOLS_UBI_IMAGE), + ) + for dependency in sorted(py3_tools_ubi_keys.difference(py3_tools_keys)) + ) + ) + discrepancies.extend( # shared dependencies with py3-tools, different versions + ( + Discrepancy( + dependency=dependency, + image=PY3_TOOLS_UBI_IMAGE, + reference_image=PY3_TOOLS_IMAGE, + in_image=py3_tools_ubi_constraints.get(dependency), + in_reference=py3_tools_constraints.get(dependency), + path=get_dependency_file_path(PY3_TOOLS_UBI_IMAGE), ) + for dependency in sorted(py3_tools_ubi_keys.intersection(py3_tools_keys)) + if py3_tools_ubi_constraints.get(dependency) + != py3_tools_constraints.get(dependency) ) + ) - for discrepancy in discrepancies: - print(str(discrepancy)) + return discrepancies def load_native_image_conf() -> list[str]: + """Returns the supported docker images by the native image from a remote JSON file.""" return json.loads( requests.get( "https://raw.githubusercontent.com/demisto/content/master/Tests/docker_native_image_config.json", @@ -106,4 +231,5 @@ def load_native_image_conf() -> list[str]: )["native_images"]["native:candidate"]["supported_docker_images"] -compare_constraints(load_native_image_conf()) +if __name__ == "__main__": + sys.exit(compare_constraints(load_native_image_conf()))