Automate management of aliases

.github/workflows/python.yml

@@ -53,6 +53,9 @@ jobs:
             ${{ steps.get-changed-files.outputs.all_changed_files }}
         run: ruff format $ALL_CHANGED_FILES
 
+      - name: check aliases
+        run: python ./python/aliases.py check
+
   unit_tests:
     name: Unit Tests
     needs: lint_stylecheck

python/aliases.py

@@ -0,0 +1,158 @@
+#!/usr/bin/python
+
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+"""Scripts for automatic management of aliases."""
+
+import importlib
+import sys
+from pathlib import Path
+
+
+SOURCES = [
+    "hopsworks/__init__.py",
+    "hopsworks/connection.py",
+    "hopsworks/internal",
+    "hopsworks/platform",
+    "hopsworks/fs",
+    "hopsworks/ml",
+]
+IGNORED = ["tests", "hsfs", "hopsworks", "hsml", "hopsworks_common"]
+# Everything that is not a top-level file, a part of sources, or a part of ignored is considered to be autmoatically managed.
+
+
+def collect_imports(root):
+    imports = []
+
+    def imports_add(file):
+        pkg = str(file.parent.relative_to(root)).replace("/", ".")
+        if file.name == "__init__.py":
+            imports.append(pkg)
+        elif file.name.endswith(".py"):
+            imports.append(pkg + "." + file.name[:-3])
+
+    for source in SOURCES:
+        if (root / source).is_file():
+            imports_add(root / source)
+            continue
+        for dirpath, _, filenames in (root / source).walk():
+            for filename in filenames:
+                imports_add(dirpath / filename)
+    return imports
+
+
+def collect_aliases(root):
+    for import_str in collect_imports(root):
+        importlib.import_module(import_str, package=".")
+    aliases = importlib.import_module("hopsworks.internal.aliases", package=".")
+    return aliases._aliases
+
+
+def collect_managed(root):
+    managed = {}
+    for pkg, from_imports in collect_aliases(root).items():
+        pkg = root / pkg.replace(".", "/") / "__init__.py"
+        managed[pkg] = (
+            "# ruff: noqa\n"
+            "# This file is generated by aliases.py. Do not edit it manually!\n"
+        )
+        from_imports.sort()  # this is needed for determinism
+        for f, i in from_imports:
+            managed[pkg] += f"from {f} import {i}\n"
+    return managed
+
+
+def fix(root):
+    managed = collect_managed(root)
+    for filepath, content in managed.items():
+        filepath.parent.mkdir(parents=True, exist_ok=True)
+        filepath.touch()
+        filepath.write_text(content)
+    ignored = [root / path for path in SOURCES + IGNORED]
+    for dirpath, _, filenames in root.walk():
+        if dirpath == root:
+            continue
+        for filename in filenames:
+            filepath = dirpath / filename
+            if any(filepath.is_relative_to(p) for p in ignored):
+                continue
+            if filepath not in managed:
+                filepath.unlink()
+
+
+def check(root):
+    ok = True
+    managed = collect_managed(root)
+    ignored = [root / path for path in SOURCES + IGNORED]
+    for dirpath, _, filenames in root.walk():
+        if dirpath == root:
+            continue
+        for filename in filenames:
+            filepath = dirpath / filename
+            if any(filepath.is_relative_to(p) for p in ignored):
+                continue
+            if filepath not in managed:
+                print(f"Error: {filepath} shouldn't exist.")
+                ok = False
+                continue
+            if filepath.read_text() != managed[filepath]:
+                print(f"Error: {filepath} has wrong content.")
+                ok = False
+    if ok:
+        print("The aliases are correct!")
+    else:
+        print("To fix the errors, run `aliases.py fix`.")
+        exit(1)
+
+
+def help(msg=None):
+    if msg:
+        print(msg + "\n")
+    print("Use `aliases.py fix [path]` or `aliases.py check [path]`.")
+    print(
+        "`path` is optional, current directory (or its `python` subdirectory) is used by default; it should be the directory containing the hopsworks package, e.g., `./python/`."
+    )
+    exit(1)
+
+
+def main():
+    if len(sys.argv) == 3:
+        root = Path(sys.argv[2])
+    elif len(sys.argv) == 2:
+        root = Path()
+        if not (root / "hopsworks").exists():
+            root = root / "python"
+    else:
+        help("Wrong number of arguments.")
+
+    root = root.resolve()
+    if not (root / "hopsworks").exists():
+        help("The used path doesn't contain the hopsworks package.")
+
+    cmd = sys.argv[1]
+    if cmd in ["f", "fix"]:
+        cmd = fix
+    elif cmd in ["c", "check"]:
+        cmd = check
+    else:
+        help("Unknown command.")
+
+    cmd(root)
+
+
+if __name__ == "__main__":
+    main()

python/hopsworks/internal/aliases.py

@@ -0,0 +1,112 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+"""Automatic management of aliases.
+
+The associated scripts are located in `python/aliases.py`.
+"""
+
+from __future__ import annotations
+
+import functools
+import inspect
+import warnings
+from typing import Optional, Tuple
+
+from hopsworks.internal.exceptions import InternalError
+
+
+_aliases = {}
+
+
+def _aliases_add(from_import: Tuple[str, str], *paths: str):
+    global _aliases
+    if "." in from_import[1]:
+        raise InternalError("Impossible to create alias for not importable symbol.")
+    for p in paths:
+        _aliases.setdefault(p, []).append(from_import)
+
+
+def public(*paths: str):
+    """Make a function or class publically available.
+
+    If you want to publish a constant, use `publish`.
+    Note that it is impossible to create an alias for a variable, i.e., it is impossible to make a change of a variable in one module to propogate to another variable in another module.
+
+    # Arguments
+        paths: the import paths under which the entity is publically avilable; effectively results in generation of aliases in all of the paths for the entity.
+    """
+
+    global publics
+
+    def decorator(symbol):
+        if not hasattr(symbol, "__qualname__"):
+            raise InternalError("The symbol should be importable to be public.")
+        _aliases_add((symbol.__module__, symbol.__qualname__), *paths)
+        return symbol
+
+    return decorator
+
+
+def publish(name: str, *paths: str):
+    """Make a constant publically available.
+
+    Since `public` decorator works only for classes and functions, this function should be used for public constants.
+    Note that it is impossible to create an alias for a variable, i.e., it is impossible to make a change of a variable in one module to propogate to another variable in another module.
+
+    # Arguments
+        name: name of the constant to be published.
+        paths: the import paths under which the names declared in the current module will be publically available; effectively results in generation of aliases in all of the paths for all the names declared in the current module.
+    """
+
+    caller = inspect.getmodule(inspect.stack()[1][0])
+
+    _aliases_add((caller.__name__, name), *paths)
+
+
+class DeprecatedCallWarning(Warning):
+    pass
+
+
+def deprecated(*, available_until: Optional[str] = None):
+    """Mark a function or class as deprecated.
+
+    Use of the entity outside hopsworks will print a warning, saying that it is going to be removed from the public API in one of the future releases.
+
+    # Arguments
+        available_until: the first hopsworks release in which the entity will become unavailable, defaults to `None`; if the release is known, it is reoprted to the external user in the warning.
+    """
+
+    v = f"version {available_until}" if available_until else "a future release"
+
+    def decorator(symbol):
+        if inspect.isclass(symbol):
+
+            @functools.wraps(symbol)
+            def decorated_f(*args, **kwargs):
+                caller = inspect.getmodule(inspect.stack()[1][0])
+                if not caller or not caller.__name__.startswith("hopsworks"):
+                    warnings.warn(
+                        f"Use of {symbol.__qualname__} is deprecated."
+                        f"The function will be removed in {v} of hopsworks.",
+                        DeprecatedCallWarning,
+                        stacklevel=2,
+                    )
+                return symbol(*args, **kwargs)
+
+            return decorated_f
+
+    return decorator

python/hopsworks/internal/exceptions.py

@@ -0,0 +1,25 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+
+class InternalError(Exception):
+    """Internal hopsworks exception.
+
+    This is raised in cases when the user of hopsworks cannot be blaimed for the error.
+    Ideally, this exception should never happen, as it means that one of the hopsworks contributors commited an error.
+
+    For example, this exception can be thrown if an internally called function which works only with `str` was given a `float`, or if a public alias is requested for a method of a class.
+    """