From 70757394759640428a0ee219fc2ae9f0a5a2bd0c Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Tue, 27 Aug 2024 16:43:49 +0200 Subject: [PATCH] Automate management of aliases --- .github/workflows/python.yml | 3 + python/aliases.py | 158 ++++++++++++++++++++++++ python/hopsworks/internal/aliases.py | 112 +++++++++++++++++ python/hopsworks/internal/exceptions.py | 25 ++++ 4 files changed, 298 insertions(+) create mode 100755 python/aliases.py create mode 100644 python/hopsworks/internal/aliases.py create mode 100644 python/hopsworks/internal/exceptions.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 9ad3513ad..5a950eafe 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -53,6 +53,9 @@ jobs: ${{ steps.get-changed-files.outputs.all_changed_files }} run: ruff format $ALL_CHANGED_FILES + - name: check aliases + run: python ./python/aliases.py check + unit_tests: name: Unit Tests needs: lint_stylecheck diff --git a/python/aliases.py b/python/aliases.py new file mode 100755 index 000000000..d8366f252 --- /dev/null +++ b/python/aliases.py @@ -0,0 +1,158 @@ +#!/usr/bin/python + +# +# Copyright 2024 Hopsworks AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Scripts for automatic management of aliases.""" + +import importlib +import sys +from pathlib import Path + + +SOURCES = [ + "hopsworks/__init__.py", + "hopsworks/connection.py", + "hopsworks/internal", + "hopsworks/platform", + "hopsworks/fs", + "hopsworks/ml", +] +IGNORED = ["tests", "hsfs", "hopsworks", "hsml", "hopsworks_common"] +# Everything that is not a top-level file, a part of sources, or a part of ignored is considered to be autmoatically managed. + + +def collect_imports(root): + imports = [] + + def imports_add(file): + pkg = str(file.parent.relative_to(root)).replace("/", ".") + if file.name == "__init__.py": + imports.append(pkg) + elif file.name.endswith(".py"): + imports.append(pkg + "." + file.name[:-3]) + + for source in SOURCES: + if (root / source).is_file(): + imports_add(root / source) + continue + for dirpath, _, filenames in (root / source).walk(): + for filename in filenames: + imports_add(dirpath / filename) + return imports + + +def collect_aliases(root): + for import_str in collect_imports(root): + importlib.import_module(import_str, package=".") + aliases = importlib.import_module("hopsworks.internal.aliases", package=".") + return aliases._aliases + + +def collect_managed(root): + managed = {} + for pkg, from_imports in collect_aliases(root).items(): + pkg = root / pkg.replace(".", "/") / "__init__.py" + managed[pkg] = ( + "# ruff: noqa\n" + "# This file is generated by aliases.py. Do not edit it manually!\n" + ) + from_imports.sort() # this is needed for determinism + for f, i in from_imports: + managed[pkg] += f"from {f} import {i}\n" + return managed + + +def fix(root): + managed = collect_managed(root) + for filepath, content in managed.items(): + filepath.parent.mkdir(parents=True, exist_ok=True) + filepath.touch() + filepath.write_text(content) + ignored = [root / path for path in SOURCES + IGNORED] + for dirpath, _, filenames in root.walk(): + if dirpath == root: + continue + for filename in filenames: + filepath = dirpath / filename + if any(filepath.is_relative_to(p) for p in ignored): + continue + if filepath not in managed: + filepath.unlink() + + +def check(root): + ok = True + managed = collect_managed(root) + ignored = [root / path for path in SOURCES + IGNORED] + for dirpath, _, filenames in root.walk(): + if dirpath == root: + continue + for filename in filenames: + filepath = dirpath / filename + if any(filepath.is_relative_to(p) for p in ignored): + continue + if filepath not in managed: + print(f"Error: {filepath} shouldn't exist.") + ok = False + continue + if filepath.read_text() != managed[filepath]: + print(f"Error: {filepath} has wrong content.") + ok = False + if ok: + print("The aliases are correct!") + else: + print("To fix the errors, run `aliases.py fix`.") + exit(1) + + +def help(msg=None): + if msg: + print(msg + "\n") + print("Use `aliases.py fix [path]` or `aliases.py check [path]`.") + print( + "`path` is optional, current directory (or its `python` subdirectory) is used by default; it should be the directory containing the hopsworks package, e.g., `./python/`." + ) + exit(1) + + +def main(): + if len(sys.argv) == 3: + root = Path(sys.argv[2]) + elif len(sys.argv) == 2: + root = Path() + if not (root / "hopsworks").exists(): + root = root / "python" + else: + help("Wrong number of arguments.") + + root = root.resolve() + if not (root / "hopsworks").exists(): + help("The used path doesn't contain the hopsworks package.") + + cmd = sys.argv[1] + if cmd in ["f", "fix"]: + cmd = fix + elif cmd in ["c", "check"]: + cmd = check + else: + help("Unknown command.") + + cmd(root) + + +if __name__ == "__main__": + main() diff --git a/python/hopsworks/internal/aliases.py b/python/hopsworks/internal/aliases.py new file mode 100644 index 000000000..17d7e9c5d --- /dev/null +++ b/python/hopsworks/internal/aliases.py @@ -0,0 +1,112 @@ +# +# Copyright 2024 Hopsworks AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Automatic management of aliases. + +The associated scripts are located in `python/aliases.py`. +""" + +from __future__ import annotations + +import functools +import inspect +import warnings +from typing import Optional, Tuple + +from hopsworks.internal.exceptions import InternalError + + +_aliases = {} + + +def _aliases_add(from_import: Tuple[str, str], *paths: str): + global _aliases + if "." in from_import[1]: + raise InternalError("Impossible to create alias for not importable symbol.") + for p in paths: + _aliases.setdefault(p, []).append(from_import) + + +def public(*paths: str): + """Make a function or class publically available. + + If you want to publish a constant, use `publish`. + Note that it is impossible to create an alias for a variable, i.e., it is impossible to make a change of a variable in one module to propogate to another variable in another module. + + # Arguments + paths: the import paths under which the entity is publically avilable; effectively results in generation of aliases in all of the paths for the entity. + """ + + global publics + + def decorator(symbol): + if not hasattr(symbol, "__qualname__"): + raise InternalError("The symbol should be importable to be public.") + _aliases_add((symbol.__module__, symbol.__qualname__), *paths) + return symbol + + return decorator + + +def publish(name: str, *paths: str): + """Make a constant publically available. + + Since `public` decorator works only for classes and functions, this function should be used for public constants. + Note that it is impossible to create an alias for a variable, i.e., it is impossible to make a change of a variable in one module to propogate to another variable in another module. + + # Arguments + name: name of the constant to be published. + paths: the import paths under which the names declared in the current module will be publically available; effectively results in generation of aliases in all of the paths for all the names declared in the current module. + """ + + caller = inspect.getmodule(inspect.stack()[1][0]) + + _aliases_add((caller.__name__, name), *paths) + + +class DeprecatedCallWarning(Warning): + pass + + +def deprecated(*, available_until: Optional[str] = None): + """Mark a function or class as deprecated. + + Use of the entity outside hopsworks will print a warning, saying that it is going to be removed from the public API in one of the future releases. + + # Arguments + available_until: the first hopsworks release in which the entity will become unavailable, defaults to `None`; if the release is known, it is reoprted to the external user in the warning. + """ + + v = f"version {available_until}" if available_until else "a future release" + + def decorator(symbol): + if inspect.isclass(symbol): + + @functools.wraps(symbol) + def decorated_f(*args, **kwargs): + caller = inspect.getmodule(inspect.stack()[1][0]) + if not caller or not caller.__name__.startswith("hopsworks"): + warnings.warn( + f"Use of {symbol.__qualname__} is deprecated." + f"The function will be removed in {v} of hopsworks.", + DeprecatedCallWarning, + stacklevel=2, + ) + return symbol(*args, **kwargs) + + return decorated_f + + return decorator diff --git a/python/hopsworks/internal/exceptions.py b/python/hopsworks/internal/exceptions.py new file mode 100644 index 000000000..443ba7d14 --- /dev/null +++ b/python/hopsworks/internal/exceptions.py @@ -0,0 +1,25 @@ +# +# Copyright 2024 Hopsworks AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class InternalError(Exception): + """Internal hopsworks exception. + + This is raised in cases when the user of hopsworks cannot be blaimed for the error. + Ideally, this exception should never happen, as it means that one of the hopsworks contributors commited an error. + + For example, this exception can be thrown if an internally called function which works only with `str` was given a `float`, or if a public alias is requested for a method of a class. + """