From f1da713e2ec6ba6e8af9680afee6bdc05809e83a Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 9 Jan 2025 11:52:34 +0100 Subject: [PATCH 1/8] add setuptools as dev dependency https://github.com/nornir-automation/nornir/issues/917 nornir should add this as a dependency which will happen in the new 3.5.0 release --- poetry.lock | 10 ++++++++++ pyproject.toml | 1 + 2 files changed, 11 insertions(+) diff --git a/poetry.lock b/poetry.lock index b8fc07f..83c19e2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1030,6 +1030,16 @@ files = [ {file = "ruff-0.3.3.tar.gz", hash = "sha256:38671be06f57a2f8aba957d9f701ea889aa5736be806f18c0cd03d6ff0cbca8d"}, ] +[[package]] +name = "setuptools" +version = "75.7.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.9" +files = [ + {file = "setuptools-75.7.0-py3-none-any.whl", hash = "sha256:84fb203f278ebcf5cd08f97d3fb96d3fbed4b629d500b29ad60d11e00769b183"}, + {file = "setuptools-75.7.0.tar.gz", hash = "sha256:886ff7b16cd342f1d1defc16fc98c9ce3fde69e087a4e1983d7ab634e5f41f4f"}, +] [[package]] name = "six" version = "1.16.0" diff --git a/pyproject.toml b/pyproject.toml index d8fdf47..91a32ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ pytest-asyncio = "^0.21.1" types-python-slugify = "*" invoke = "2.2.0" yamllint = "*" +setuptools = "^75.7.0" [tool.coverage.run] branch = true From 9e51c4db5bff49e056f176b8a43db94e94702d78 Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 9 Jan 2025 11:54:19 +0100 Subject: [PATCH 2/8] bump infrahub-sdk to v1.4.1 and set min version to v1.3.0 v1.3.0 introduces new pydantic models for different schema related models that we need to use in this project --- poetry.lock | 39 ++++++++++++++++++++++++++++++++------- pyproject.toml | 2 +- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 83c19e2..83da7d6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "annotated-types" @@ -55,6 +55,20 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "eval-type-backport" +version = "0.2.2" +description = "Like `typing._eval_type`, but lets older Python versions use newer typing features." +optional = false +python-versions = ">=3.8" +files = [ + {file = "eval_type_backport-0.2.2-py3-none-any.whl", hash = "sha256:cb6ad7c393517f476f96d456d0412ea80f0a8cf96f6892834cd9340149111b0a"}, + {file = "eval_type_backport-0.2.2.tar.gz", hash = "sha256:f0576b4cf01ebb5bd358d02314d31846af5e07678387486e2c798af0e7d849c1"}, +] + +[package.extras] +tests = ["pytest"] + [[package]] name = "exceptiongroup" version = "1.2.2" @@ -207,16 +221,17 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag [[package]] name = "infrahub-sdk" -version = "1.1.0" +version = "1.4.1" description = "Python Client to interact with Infrahub" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "infrahub_sdk-1.1.0-py3-none-any.whl", hash = "sha256:b996a6d5d862e66f13a8dc68adad74a59a1f1175a33809db5e84fc58276ebf35"}, - {file = "infrahub_sdk-1.1.0.tar.gz", hash = "sha256:b0f995053a6a92f72ec92abef2564080ff9c72e026db1bc0d686cb84ffd1e5e0"}, + {file = "infrahub_sdk-1.4.1-py3-none-any.whl", hash = "sha256:cb1f409042c3ad539876e2a2d0a294e9acf6d42164ba218571a156322b125adf"}, + {file = "infrahub_sdk-1.4.1.tar.gz", hash = "sha256:115ed24cf4198a26c6524d131405dd1b95419eb57d486a9f03db490e9de5b501"}, ] [package.dependencies] +eval-type-backport = {version = ">=0.2.2,<0.3.0", markers = "python_version >= \"3.9\" and python_version < \"3.10\""} gitpython = ">=3,<4" graphql-core = ">=3.1,<3.3" httpx = [ @@ -236,8 +251,8 @@ rich = {version = ">=13,<14", optional = true, markers = "extra == \"ctl\" or ex ujson = ">=5,<6" [package.extras] -all = ["Jinja2 (>=3,<4)", "numpy (>=1.24.2,<2.0.0)", "numpy (>=1.26.2,<2.0.0)", "pyarrow (>=14,<15)", "pytest", "pyyaml (>=6,<7)", "rich (>=13,<14)", "toml (>=0.10,<0.11)", "typer (>=0.12.3,<0.13.0)"] -ctl = ["Jinja2 (>=3,<4)", "numpy (>=1.24.2,<2.0.0)", "numpy (>=1.26.2,<2.0.0)", "pyarrow (>=14,<15)", "pyyaml (>=6,<7)", "rich (>=13,<14)", "toml (>=0.10,<0.11)", "typer (>=0.12.3,<0.13.0)"] +all = ["Jinja2 (>=3,<4)", "numpy (>=1.24.2,<2.0.0)", "numpy (>=1.26.2,<2.0.0)", "pyarrow (>=14)", "pytest", "pyyaml (>=6,<7)", "rich (>=13,<14)", "toml (>=0.10,<0.11)", "typer (>=0.12.3,<0.13.0)"] +ctl = ["Jinja2 (>=3,<4)", "numpy (>=1.24.2,<2.0.0)", "numpy (>=1.26.2,<2.0.0)", "pyarrow (>=14)", "pyyaml (>=6,<7)", "rich (>=13,<14)", "toml (>=0.10,<0.11)", "typer (>=0.12.3,<0.13.0)"] tests = ["Jinja2 (>=3,<4)", "pytest", "pyyaml (>=6,<7)", "rich (>=13,<14)"] [[package]] @@ -1040,6 +1055,16 @@ files = [ {file = "setuptools-75.7.0-py3-none-any.whl", hash = "sha256:84fb203f278ebcf5cd08f97d3fb96d3fbed4b629d500b29ad60d11e00769b183"}, {file = "setuptools-75.7.0.tar.gz", hash = "sha256:886ff7b16cd342f1d1defc16fc98c9ce3fde69e087a4e1983d7ab634e5f41f4f"}, ] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] +core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] + [[package]] name = "six" version = "1.16.0" @@ -1255,4 +1280,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9, <3.13" -content-hash = "89d260e210967d69d207cd3c77f04ab39176bd341d050e565100ce868c6d09d2" +content-hash = "25e7144dd8548cbb74e7a4dae8a5928ed22ebe18ae0bd6b8150f67e2de36c781" diff --git a/pyproject.toml b/pyproject.toml index 91a32ec..e99bc49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ packages = [{ include = "nornir_infrahub" }] [tool.poetry.dependencies] python = "^3.9, <3.13" -infrahub-sdk = { version = "^1", extras = ["tests"] } +infrahub-sdk = { version = "^1,>=1.3.0", extras = ["tests"] } ruamel-yaml = "^0.18.5" nornir = "^3.4.1" nornir-utils = "^0.2.0" From ca3b297a5464d67588ed9f3163972b1afee1b651 Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 9 Jan 2025 11:55:19 +0100 Subject: [PATCH 3/8] use new NodeSchemaAPI model introduced in infrahub-sdk 1.3.0 --- nornir_infrahub/plugins/inventory/infrahub.py | 4 ++-- tests/unit/conftest.py | 14 +++++++------- tests/unit/test_inventory.py | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/nornir_infrahub/plugins/inventory/infrahub.py b/nornir_infrahub/plugins/inventory/infrahub.py index 1041d59..264ec63 100644 --- a/nornir_infrahub/plugins/inventory/infrahub.py +++ b/nornir_infrahub/plugins/inventory/infrahub.py @@ -7,7 +7,7 @@ import ruamel.yaml from infrahub_sdk import Config, InfrahubClientSync from infrahub_sdk.node import InfrahubNodeSync -from infrahub_sdk.schema import NodeSchema +from infrahub_sdk.schema import NodeSchemaAPI from nornir.core.inventory import ( ConnectionOptions, Defaults, @@ -115,7 +115,7 @@ def validate_include(cls, data: Any) -> Any: return data -def get_related_nodes(node_schema: NodeSchema, attrs: Set[str]) -> Set[str]: +def get_related_nodes(node_schema: NodeSchemaAPI, attrs: Set[str]) -> Set[str]: nodes = {"CoreStandardGroup"} relationship_schemas = {schema.name: schema.peer for schema in node_schema.relationships} for attr in attrs: diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 81e554f..059f5f0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -3,7 +3,7 @@ import pytest from infrahub_sdk import InfrahubClient, InfrahubClientSync -from infrahub_sdk.schema import NodeSchema +from infrahub_sdk.schema import NodeSchemaAPI from nornir_infrahub.plugins.inventory.infrahub import InfrahubInventory @@ -18,7 +18,7 @@ async def client_sync() -> InfrahubClientSync: @pytest.fixture -async def location_schema() -> NodeSchema: +async def location_schema() -> NodeSchemaAPI: data = { "name": "Location", "namespace": "Builtin", @@ -34,11 +34,11 @@ async def location_schema() -> NodeSchema: {"name": "member_of_groups", "peer": "CoreGroup", "optional": True, "cardinality": "many", "kind": "Group"}, ], } - return NodeSchema(**data) + return NodeSchemaAPI(**data) @pytest.fixture -async def device_schema() -> NodeSchema: +async def device_schema() -> NodeSchemaAPI: data = { "name": "Device", "namespace": "Infra", @@ -68,11 +68,11 @@ async def device_schema() -> NodeSchema: }, ], } - return NodeSchema(**data) + return NodeSchemaAPI(**data) @pytest.fixture -async def ipaddress_schema() -> NodeSchema: +async def ipaddress_schema() -> NodeSchemaAPI: data = { "name": "IPAddress", "namespace": "Infra", @@ -86,7 +86,7 @@ async def ipaddress_schema() -> NodeSchema: {"name": "interface", "peer": "InfraInterfaceL3", "optional": True, "cardinality": "one", "kind": "Parent"} ], } - return NodeSchema(**data) + return NodeSchemaAPI(**data) @pytest.fixture diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py index b7070e5..518bfb4 100644 --- a/tests/unit/test_inventory.py +++ b/tests/unit/test_inventory.py @@ -4,7 +4,7 @@ import pytest from infrahub_sdk import InfrahubClient from infrahub_sdk.node import InfrahubNodeSync -from infrahub_sdk.schema import NodeSchema +from infrahub_sdk.schema import NodeSchemaAPI from nornir.core.inventory import ConnectionOptions, Defaults # , HostOrGroup from nornir_infrahub.plugins.inventory.infrahub import ( # _get_inventory_element, HostNode, @@ -38,21 +38,21 @@ def test_ip_interface_to_ip_string_ipv6(): # resolve_node_mapping -def test_valid_mapping(client: InfrahubClient, ipaddress_schema: NodeSchema, ipaddress_data: dict[str, Any]): +def test_valid_mapping(client: InfrahubClient, ipaddress_schema: NodeSchemaAPI, ipaddress_data: dict[str, Any]): node = InfrahubNodeSync(client=client, schema=ipaddress_schema, data=ipaddress_data) attrs = ["address"] result = resolve_node_mapping(node, attrs) assert result == "192.168.1.1" -def test_unsupported_cardinality(client: InfrahubClient, location_schema: NodeSchema): +def test_unsupported_cardinality(client: InfrahubClient, location_schema: NodeSchemaAPI): node = InfrahubNodeSync(client=client, schema=location_schema) attrs = ["tags"] with pytest.raises(RuntimeError, match="Relations with many cardinality are not supported!"): resolve_node_mapping(node, attrs) -def test_invalid_mapping(client: InfrahubClient, ipaddress_schema: NodeSchema): +def test_invalid_mapping(client: InfrahubClient, ipaddress_schema: NodeSchemaAPI): node = InfrahubNodeSync(client=client, schema=ipaddress_schema) attrs = ["invalid_attribute"] with pytest.raises(RuntimeError, match="Unable to resolve mapping"): @@ -287,7 +287,7 @@ def test_get_related_nodes(): {"name": "vehicules", "peer": "TestVehicule", "cardinality": "many", "identifier": "person__vehicule"} ], } - node_schema = NodeSchema(**schema) + node_schema = NodeSchemaAPI(**schema) attrs = {"vehicules", "address"} # "Address" is not in the relationships result = get_related_nodes(node_schema, attrs) expected_result = {"CoreStandardGroup", "TestVehicule"} From 5f70cac969fc40af8cf906e45d16a91f5692ec0c Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 9 Jan 2025 11:59:57 +0100 Subject: [PATCH 4/8] use setup-python@v5 in ci --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c999e05..21d096b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,8 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v3" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 + id: python with: python-version: ${{ matrix.python-version }} - name: "Setup environment" From 61d77734dcaf56b0bf88d2e855b30ffe5c7fc19a Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 9 Jan 2025 12:00:44 +0100 Subject: [PATCH 5/8] set PIPX_DEFAULT_PYTHON in CI --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21d096b..3956cd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,8 @@ jobs: run: | pipx install poetry pip install invoke toml + env: + PIPX_DEFAULT_PYTHON: ${{ steps.python.outputs.python-path }} - name: "Install Nornir Infrahub" run: "poetry install" - name: "Pytest Tests" From 3a4e55a92987e8dc1205794efba99a4f7540da3f Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 9 Jan 2025 12:01:20 +0100 Subject: [PATCH 6/8] install dev dependencies in CI --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3956cd8..5490f12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,8 +22,7 @@ jobs: uses: "actions/checkout@v3" - name: "Setup environment" run: | - pipx install poetry - pip install invoke toml + pipx install poetry invoke - name: "Install Linters" run: "poetry install --only=dev" - name: "Linting" @@ -51,11 +50,10 @@ jobs: - name: "Setup environment" run: | pipx install poetry - pip install invoke toml env: PIPX_DEFAULT_PYTHON: ${{ steps.python.outputs.python-path }} - name: "Install Nornir Infrahub" - run: "poetry install" + run: "poetry install --with=dev" - name: "Pytest Tests" run: "poetry run pytest -v tests/" From 3934057be8cb3e72113d1b3a98e4f0d0d20eaeea Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 9 Jan 2025 14:26:07 +0100 Subject: [PATCH 7/8] update tasks example --- examples/nornir_tasks.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) mode change 100644 => 100755 examples/nornir_tasks.py diff --git a/examples/nornir_tasks.py b/examples/nornir_tasks.py old mode 100644 new mode 100755 index 650bc64..23e711c --- a/examples/nornir_tasks.py +++ b/examples/nornir_tasks.py @@ -46,25 +46,31 @@ def main(): # we only need to run this task once, per artifact definition run_once = nr.filter(name="jfk1-edge1") result = run_once.run(task=generate_artifacts, artifact="startup-config", timeout=20) + ocfg_result = run_once.run(task=generate_artifacts, artifact="openconfig-interfaces", timeout=20) for _, v in result.items(): if v[0].failed: return 1 + for _, v in ocfg_result.items(): + if v[0].failed: + return 1 + # These artifacts are generated for Arista EOS devices in demo-edge + eos_devices = nr.filter(platform="eos") # retrieves the artifact for all the hosts in the inventory - result = nr.run(task=get_artifact, artifact="startup-config") + result = eos_devices.run(task=get_artifact, artifact="Startup Config for Edge devices") print_result(result) # push the retrieved artifact to a device # print_result(run_once.run(task=napalm_configure, configuration=result["jfk1-edge1"][0].result, replace=True)) # artifacts with content-type application/json get deserialized - result = nr.run(task=get_artifact, artifact="openconfig-interfaces") + result = eos_devices.run(task=get_artifact, artifact="Openconfig Interface for Arista devices") print_result(result) assert isinstance(result["den1-edge1"][0].result, dict) # regenerate an artifact for a host - print_result(nr.run(task=regenerate_host_artifact, artifact="startup-config")) + print_result(eos_devices.run(task=regenerate_host_artifact, artifact="Startup Config for Edge devices")) return 0 From 3afb81f5e02f4ba85fca12e35cf92b11a7032015 Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 9 Jan 2025 14:32:32 +0100 Subject: [PATCH 8/8] bump package version to 1.0.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e99bc49..d49b47f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "nornir-infrahub" -version = "1.0.0" +version = "1.0.1" description = "Nornir plugin for Infrahub" authors = ["OpsMill "] readme = "README.md"