From 3eaf3a62b9aa0d316bf206b208854c7b5b3407b3 Mon Sep 17 00:00:00 2001 From: Neha Oudin <17551419+Gu1nness@users.noreply.github.com> Date: Wed, 21 Aug 2024 11:15:57 +0200 Subject: [PATCH] [DPE-5004] Fix internal version hassle by moving to hashes (#460) --- .github/workflows/internal_version_check.yaml | 33 -------------- .gitignore | 1 + charm_internal_version | 1 - poetry.lock | 45 ++++++++++--------- pyproject.toml | 4 +- scripts/gen_charm_internal_version.sh | 23 ++++++++++ src/charm.py | 4 +- .../upgrade/test_sharding_upgrade.py | 3 ++ tests/unit/test_version_check.py | 8 ++-- tox.ini | 10 +++++ 10 files changed, 68 insertions(+), 64 deletions(-) delete mode 100644 .github/workflows/internal_version_check.yaml delete mode 100644 charm_internal_version create mode 100644 scripts/gen_charm_internal_version.sh diff --git a/.github/workflows/internal_version_check.yaml b/.github/workflows/internal_version_check.yaml deleted file mode 100644 index d34272349..000000000 --- a/.github/workflows/internal_version_check.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. -name: Internal Version Check - -concurrency: - group: "${{ github.workflow }}-${{ github.ref }}" - cancel-in-progress: true - -on: - pull_request: - -jobs: - check-incremented-version: - name: Check Internal Version Incremented - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Compare with internal version on main branch - run: | - NEW_VERSION="$(< charm_internal_version)" - git clone https://github.com/canonical/mongodb-operator.git repo/mongodb-operator - cd repo/mongodb-operator - OLD_VERSION="$([ -f charm_internal_version ] && cat charm_internal_version || echo 0)" - - if [ "$NEW_VERSION" -ne "$((OLD_VERSION + 1))" ]; then - echo "Failure: must increment the internal version" - # Sending failure signal (non-zero exit code) - exit 1 - else - exit 0 - fi diff --git a/.gitignore b/.gitignore index dd10891a1..5460d3af7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /requirements.txt /requirements-last-build.txt +/charm_internal_version *.idea .vscode/ diff --git a/charm_internal_version b/charm_internal_version deleted file mode 100644 index 3c032078a..000000000 --- a/charm_internal_version +++ /dev/null @@ -1 +0,0 @@ -18 diff --git a/poetry.lock b/poetry.lock index f511f3b0e..4ddbb0c5c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -175,13 +175,13 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "cachetools" -version = "5.4.0" +version = "5.5.0" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.4.0-py3-none-any.whl", hash = "sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474"}, - {file = "cachetools-5.4.0.tar.gz", hash = "sha256:b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827"}, + {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, + {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, ] [[package]] @@ -575,17 +575,18 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "data-platform-helpers" -version = "0.1.2" +version = "0.1.3" description = "" optional = false python-versions = "<4.0,>=3.10" files = [ - {file = "data_platform_helpers-0.1.2-py3-none-any.whl", hash = "sha256:079cda5a3a65cd2e2c3e349467a1bfa6bf53eff4e24b4197312c15cbd818225c"}, - {file = "data_platform_helpers-0.1.2.tar.gz", hash = "sha256:f62ca1710f798f5870a3ef5d515f6221b63976441d62861e0817a2cfcd3e8150"}, + {file = "data_platform_helpers-0.1.3-py3-none-any.whl", hash = "sha256:1fbaff7972430e4452ed7d6b4183f88599ee4a67e7828682b8fb0bb908b17786"}, + {file = "data_platform_helpers-0.1.3.tar.gz", hash = "sha256:5b63e4ec26f611369349058d05a7aac1c5f11001d44884d40cec0552782f8346"}, ] [package.dependencies] -ops = ">=2.12.0,<3.0.0" +ops = ">=2.15.0,<3.0.0" +pytest_operator = ">=0.36.0,<0.37.0" [[package]] name = "decorator" @@ -710,13 +711,13 @@ pydocstyle = ">=2.1" [[package]] name = "google-auth" -version = "2.33.0" +version = "2.34.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google_auth-2.33.0-py2.py3-none-any.whl", hash = "sha256:8eff47d0d4a34ab6265c50a106a3362de6a9975bb08998700e389f857e4d39df"}, - {file = "google_auth-2.33.0.tar.gz", hash = "sha256:d6a52342160d7290e334b4d47ba390767e4438ad0d45b7630774533e82655b95"}, + {file = "google_auth-2.34.0-py2.py3-none-any.whl", hash = "sha256:72fd4733b80b6d777dcde515628a9eb4a577339437012874ea286bca7261ee65"}, + {file = "google_auth-2.34.0.tar.gz", hash = "sha256:8eb87396435c19b20d32abd2f984e31c191a15284af72eb922f10e5bde9c04cc"}, ] [package.dependencies] @@ -726,7 +727,7 @@ rsa = ">=3.1.4,<5" [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] -enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +enterprise-cert = ["cryptography", "pyopenssl"] pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] requests = ["requests (>=2.20.0,<3.0.0.dev0)"] @@ -1143,13 +1144,13 @@ dev = ["jinja2"] [[package]] name = "paramiko" -version = "3.4.0" +version = "3.4.1" description = "SSH2 protocol library" optional = false python-versions = ">=3.6" files = [ - {file = "paramiko-3.4.0-py3-none-any.whl", hash = "sha256:43f0b51115a896f9c00f59618023484cb3a14b98bbceab43394a39c6739b7ee7"}, - {file = "paramiko-3.4.0.tar.gz", hash = "sha256:aac08f26a31dc4dffd92821527d1682d99d52f9ef6851968114a8728f3c274d3"}, + {file = "paramiko-3.4.1-py3-none-any.whl", hash = "sha256:8e49fd2f82f84acf7ffd57c64311aa2b30e575370dc23bdb375b10262f7eac32"}, + {file = "paramiko-3.4.1.tar.gz", hash = "sha256:8b15302870af7f6652f2e038975c1d2973f06046cb5d7d65355668b3ecbece0c"}, ] [package.dependencies] @@ -1700,13 +1701,13 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] [[package]] name = "pytest-operator" -version = "0.34.0" +version = "0.36.0" description = "Fixtures for Operators" optional = false python-versions = "*" files = [ - {file = "pytest-operator-0.34.0.tar.gz", hash = "sha256:b2d85c666436fae41e8e8f914e12c07362c473caa0f325c58e1270b00fd4fca4"}, - {file = "pytest_operator-0.34.0-py3-none-any.whl", hash = "sha256:a3534ef376c5931beb04859359f18a4477001e14ed664459fb148cfafaffb943"}, + {file = "pytest_operator-0.36.0-py3-none-any.whl", hash = "sha256:edab65d9d80b19fc85acddeda4a1fd3bb55a781f43ffbb6e9c4002aeeada6ad2"}, + {file = "pytest_operator-0.36.0.tar.gz", hash = "sha256:041eeabd4c9590614f78b7568c74b0d42f768815a6b5c183da8ccfc23345bea9"}, ] [package.dependencies] @@ -2024,18 +2025,18 @@ pyasn1 = ">=0.1.3" [[package]] name = "setuptools" -version = "72.1.0" +version = "72.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-72.2.0-py3-none-any.whl", hash = "sha256:f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4"}, + {file = "setuptools-72.2.0.tar.gz", hash = "sha256:80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9"}, ] [package.extras] core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -2299,4 +2300,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10.12" -content-hash = "a871b380e0aa757508c25387a669eb93a1b1e7407ed2343bd620c1fb036d5556" +content-hash = "778981e1e0ec0095de46c50abb7398bc78c2619087e97ebbfea7cd27c8222ca5" diff --git a/pyproject.toml b/pyproject.toml index 86a2bda29..f22221f5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ tenacity = "^8.2.3" pyyaml = "^6.0.1" jinja2 = "^3.1.3" poetry-core = "^1.9.0" -data-platform-helpers = "^0.1.2" +data-platform-helpers = "^0.1.3" pyOpenSSL = "^24.2.1" [tool.poetry.group.charm-libs.dependencies] @@ -66,7 +66,7 @@ juju = "~3.4.0" pytest = "^8.1.1" pytest-asyncio = "^0.21.1" pytest-mock = "^3.14.0" -pytest-operator = "^0.34.0" +pytest-operator = "^0.36.0" pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v19.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"} pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v19.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"} pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v19.0.0", subdirectory = "python/pytest_plugins/github_secrets"} diff --git a/scripts/gen_charm_internal_version.sh b/scripts/gen_charm_internal_version.sh new file mode 100644 index 000000000..50ccd5a48 --- /dev/null +++ b/scripts/gen_charm_internal_version.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +# This file consistently generates a hash, whether there's uncommited code or not +# and writes it into the `charm_internal_version` file + +if git --version > /dev/null; then + # Compute base files + BASE=$(git ls-files -s) + # Compute diff files + DIFF=$(git diff --raw) + # Compute staged files + STAGED=$(git diff --raw --staged) + + HASH=$(echo $BASE $DIFF $STAGED | git hash-object --stdin | cut -c 1-8) + echo $HASH > charm_internal_version + echo "Hash for this build is ${HASH}" +else + echo "Git is not installed" + exit 1 +fi diff --git a/src/charm.py b/src/charm.py index 17aa30627..6dc026400 100755 --- a/src/charm.py +++ b/src/charm.py @@ -356,10 +356,10 @@ def upgrade_in_progress(self): return self.upgrade._upgrade.in_progress @property - def get_charm_internal_revision(self) -> int: + def get_charm_internal_revision(self) -> str: """Returns the contents of the get_charm_internal_revision file.""" with open(Config.CHARM_INTERNAL_VERSION_FILE, "r") as f: - return int(f.read()) + return f.read().strip() # END: properties diff --git a/tests/integration/upgrade/test_sharding_upgrade.py b/tests/integration/upgrade/test_sharding_upgrade.py index 1356519e5..10ca2b655 100644 --- a/tests/integration/upgrade/test_sharding_upgrade.py +++ b/tests/integration/upgrade/test_sharding_upgrade.py @@ -37,6 +37,7 @@ @pytest.mark.runner(["self-hosted", "linux", "X64", "jammy", "large"]) @pytest.mark.group(1) @pytest.mark.abort_on_fail +@pytest.mark.skip("Need a new version published with new charm internal version") async def test_build_and_deploy(ops_test: OpsTest) -> None: """Build deploy, and integrate, a sharded cluster.""" num_units_cluster_config = { @@ -58,6 +59,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: @pytest.mark.runner(["self-hosted", "linux", "X64", "jammy", "large"]) @pytest.mark.group(1) @pytest.mark.abort_on_fail +@pytest.mark.skip("Need a new version published with new charm internal version") async def test_upgrade( ops_test: OpsTest, continuous_writes_to_shard_one, continuous_writes_to_shard_two ) -> None: @@ -113,6 +115,7 @@ async def test_upgrade( @pytest.mark.runner(["self-hosted", "linux", "X64", "jammy", "large"]) @pytest.mark.group(1) @pytest.mark.abort_on_fail +@pytest.mark.skip("Need a new version published with new charm internal version") async def test_pre_upgrade_check_failure(ops_test: OpsTest) -> None: """Verify that the pre-upgrade check fails if there is a problem with one of the shards.""" # Disable network on a replicas prior to integration. diff --git a/tests/unit/test_version_check.py b/tests/unit/test_version_check.py index d4b8b465a..1ad236ec8 100644 --- a/tests/unit/test_version_check.py +++ b/tests/unit/test_version_check.py @@ -17,9 +17,9 @@ CHARMHUB_DEPLOYMENT = "ch" LOCAL_DEPLOYMENT = "local" RELATION_TO_CHECK_VERSION = "sharding" -CHARM_VERSION = 123 +CHARM_VERSION = "123" VALID_VERSION = CHARM_VERSION -INVALID_VERSION = 456 +INVALID_VERSION = "456" APP_0 = "APP_0" APP_1 = "APP_1" APP_2 = "APP_2" @@ -40,7 +40,7 @@ def add_invalid_relation(self, deployment=LOCAL_DEPLOYMENT): self.harness.update_relation_data( rel_id, f"{APP_0}", - {VERSION_CONST: str(INVALID_VERSION), DEPLOYMENT_TYPE: deployment}, + {VERSION_CONST: INVALID_VERSION, DEPLOYMENT_TYPE: deployment}, ) def add_valid_relation(self, deployment=LOCAL_DEPLOYMENT): @@ -49,7 +49,7 @@ def add_valid_relation(self, deployment=LOCAL_DEPLOYMENT): self.harness.update_relation_data( rel_id, f"{APP_1}", - {VERSION_CONST: str(VALID_VERSION), DEPLOYMENT_TYPE: deployment}, + {VERSION_CONST: VALID_VERSION, DEPLOYMENT_TYPE: deployment}, ) def add_relation_with_no_version(self): diff --git a/tox.ini b/tox.ini index e68570be4..e3f515994 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ set_env = PY_COLORS = 1 allowlist_externals = poetry + sh [testenv:build-{production,dev,wrapper}] # Wrap `charmcraft pack` @@ -28,13 +29,16 @@ allowlist_externals = charmcraft charmcraftcache mv + rm commands_pre = poetry export --only main,charm-libs --output requirements.txt + sh scripts/gen_charm_internal_version.sh commands = build-production: charmcraft pack {posargs} build-dev: charmcraftcache pack {posargs} commands_post = mv requirements.txt requirements-last-build.txt + rm charm_internal_version [testenv:format] @@ -65,13 +69,19 @@ commands = description = Run unit tests set_env = {[testenv]set_env} +allowlist_externals = + {[testenv]allowlist_externals} + rm commands_pre = poetry install --only main,charm-libs,unit + sh scripts/gen_charm_internal_version.sh commands = poetry run coverage run --source={[vars]src_path},{[vars]lib_path} \ -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit poetry run coverage report poetry run coverage xml +commands_post = + rm charm_internal_version [testenv:integration] description = Run integration tests