From 15ccb108980ae0bc73c78c789058b8231e5ad37e Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 12 Feb 2025 10:12:56 +0100
Subject: [PATCH 01/35] Resolve workunit_ref to absolute path if it is a Path
 instance

---
 bfabric_app_runner/docs/changelog.md                        | 4 ++++
 .../src/bfabric_app_runner/app_runner/resolve_app.py        | 6 ++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/bfabric_app_runner/docs/changelog.md b/bfabric_app_runner/docs/changelog.md
index 9533153c..87200f1b 100644
--- a/bfabric_app_runner/docs/changelog.md
+++ b/bfabric_app_runner/docs/changelog.md
@@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 - Implement `--force-storage` to pass a yaml to a forced storage instead of the real one.
 
+### Changed
+
+- Resolve workunit_ref to absolute path if it is a Path instance for CLI.
+
 ## \[0.0.15\] - 2025-02-06
 
 ### Added
diff --git a/bfabric_app_runner/src/bfabric_app_runner/app_runner/resolve_app.py b/bfabric_app_runner/src/bfabric_app_runner/app_runner/resolve_app.py
index 41ba3ae9..50f2fdf2 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/app_runner/resolve_app.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/app_runner/resolve_app.py
@@ -1,16 +1,16 @@
 from __future__ import annotations
 
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 import yaml
 from pydantic import ValidationError
 
+from bfabric.experimental.workunit_definition import WorkunitDefinition
 from bfabric_app_runner.specs.app.app_spec import AppSpec
 from bfabric_app_runner.specs.app.app_version import AppVersion
-from bfabric.experimental.workunit_definition import WorkunitDefinition
 
 if TYPE_CHECKING:
-    from pathlib import Path
     from bfabric import Bfabric
 
 
@@ -57,6 +57,8 @@ def load_workunit_information(
         steps to avoid unnecessary B-Fabric lookups. (If the workunit_ref was already a path, it will be returned as is,
         otherwise the file will be created in the work directory.)
     """
+    if isinstance(workunit_ref, Path):
+        workunit_ref = workunit_ref.resolve()
     workunit_definition_file = work_dir / "workunit_definition.yml"
     workunit_definition = WorkunitDefinition.from_ref(workunit_ref, client, cache_file=workunit_definition_file)
     app_parsed = _load_spec(

From 7a449a7bf6df6125994e2033ecd66db6b50ae57d Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leoschwarz@users.noreply.github.com>
Date: Wed, 12 Feb 2025 17:13:47 +0100
Subject: [PATCH 02/35] Add `api update` command (#142)

---
 bfabric_scripts/doc/changelog.md              |  4 ++
 .../bfabric_scripts/cli/api/cli_api_read.py   |  2 +-
 .../bfabric_scripts/cli/api/cli_api_update.py | 71 +++++++++++++++++++
 .../src/bfabric_scripts/cli/cli_api.py        |  2 +
 4 files changed, 78 insertions(+), 1 deletion(-)
 create mode 100644 bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_update.py

diff --git a/bfabric_scripts/doc/changelog.md b/bfabric_scripts/doc/changelog.md
index c5f0abeb..016abeae 100644
--- a/bfabric_scripts/doc/changelog.md
+++ b/bfabric_scripts/doc/changelog.md
@@ -16,6 +16,10 @@ Versioning currently follows `X.Y.Z` where
 
 - Add missing default value for columns in `bfabric-cli api read`
 
+### Added
+
+- `bfabric-cli api update` command to update an existing entity
+
 ## \[1.13.20\] - 2025-02-10
 
 ### Added
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py
index 94d71a5b..65af0172 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py
@@ -101,7 +101,7 @@ def render_output(results: list[dict[str, Any]], params: Params, client: Bfabric
 
 @app.default
 @use_client
-@logger.catch()
+@logger.catch(reraise=True)
 def read(params: Annotated[Params, cyclopts.Parameter(name="*")], *, client: Bfabric) -> None | int:
     """Reads one type of entity from B-Fabric."""
     console_user = Console(stderr=True)
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_update.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_update.py
new file mode 100644
index 00000000..38b17682
--- /dev/null
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_update.py
@@ -0,0 +1,71 @@
+import cyclopts
+import rich
+import rich.prompt
+from cyclopts import Parameter
+from loguru import logger
+from pydantic import BaseModel
+from rich.panel import Panel
+from rich.pretty import Pretty, pprint
+
+from bfabric import Bfabric
+from bfabric.utils.cli_integration import use_client
+
+app = cyclopts.App()
+
+
+@Parameter(name="*")
+class Params(BaseModel):
+    endpoint: str
+    """Endpoint to update, e.g. 'resource'."""
+    entity_id: int
+    """ID of the entity to update."""
+    attributes: list[tuple[str, str]] | None = None
+    """List of attribute-value pairs to update the entity with."""
+    no_confirm: bool = False
+    """If set, the update will be performed without asking for confirmation."""
+
+
+@app.default
+@use_client
+@logger.catch(reraise=True)
+def update(params: Params, *, client: Bfabric) -> None:
+    """Updates an existing entity in B-Fabric."""
+    attributes_dict = _sanitize_attributes(params.attributes, params.entity_id)
+    if not attributes_dict:
+        return
+
+    if not params.no_confirm:
+        if not _confirm_action(attributes_dict, client, params.endpoint, params.entity_id):
+            return
+
+    result = client.save(params.endpoint, {"id": params.entity_id, **attributes_dict})
+    logger.info(f"Entity with ID {params.entity_id} updated successfully.")
+    pprint(result)
+
+
+def _confirm_action(attributes_dict: dict[str, str], client: Bfabric, endpoint: str, entity_id: int) -> bool:
+    logger.info(f"Updating {endpoint} entity with ID {entity_id} with attributes {attributes_dict}")
+    result_read = client.read(endpoint, {"id": entity_id}, max_results=1)
+    if not result_read:
+        raise ValueError(f"No entity found with ID {entity_id}")
+    rich.print(Panel.fit(Pretty(result_read[0], expand_all=False), title="Existing entity"))
+    rich.print(Panel.fit(Pretty(attributes_dict, expand_all=False), title="Updates"))
+    if not rich.prompt.Confirm.ask("Do you want to proceed with the update?"):
+        logger.info("Update cancelled by user.")
+        return False
+    return True
+
+
+def _sanitize_attributes(attributes: list[tuple[str, str]] | None, entity_id: int) -> dict[str, str] | None:
+    if not attributes:
+        logger.warning("No attributes provided, doing nothing.")
+        return None
+
+    attributes_dict = {attribute: value for attribute, value in attributes}
+    if "id" in attributes_dict:
+        if int(attributes_dict["id"]) == entity_id:
+            logger.warning("Attribute 'id' is not allowed in the attributes, removing it.")
+            del attributes_dict["id"]
+        else:
+            raise ValueError("Attribute 'id' must match the entity_id")
+    return attributes_dict
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
index 4d7c2522..9dc0cc46 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
@@ -3,8 +3,10 @@
 from bfabric_scripts.cli.api.cli_api_log import cmd as _cmd_log
 from bfabric_scripts.cli.api.cli_api_read import app as _cmd_read
 from bfabric_scripts.cli.api.cli_api_save import app as _cmd_save
+from bfabric_scripts.cli.api.cli_api_update import app as _cmd_update
 
 app = cyclopts.App()
 app.command(_cmd_log, name="log")
 app.command(_cmd_read, name="read")
+app.command(_cmd_update, name="update")
 app.command(_cmd_save, name="save")

From 9d2370a900abb00b8480d7bbc4b21fce5302fd3e Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 08:49:01 +0100
Subject: [PATCH 03/35] Add `bfabric-cli api delete` command to delete an
 existing entity

---
 bfabric_scripts/doc/changelog.md              |  1 +
 .../bfabric_scripts/cli/api/cli_api_delete.py | 55 +++++++++++++++++++
 .../src/bfabric_scripts/cli/cli_api.py        |  7 ++-
 3 files changed, 62 insertions(+), 1 deletion(-)
 create mode 100644 bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_delete.py

diff --git a/bfabric_scripts/doc/changelog.md b/bfabric_scripts/doc/changelog.md
index 016abeae..87d58b7b 100644
--- a/bfabric_scripts/doc/changelog.md
+++ b/bfabric_scripts/doc/changelog.md
@@ -19,6 +19,7 @@ Versioning currently follows `X.Y.Z` where
 ### Added
 
 - `bfabric-cli api update` command to update an existing entity
+- `bfabric-cli api delete` command to delete an existing entity
 
 ## \[1.13.20\] - 2025-02-10
 
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_delete.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_delete.py
new file mode 100644
index 00000000..341687b6
--- /dev/null
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_delete.py
@@ -0,0 +1,55 @@
+import cyclopts
+import rich
+from loguru import logger
+from pydantic import BaseModel
+from rich.panel import Panel
+from rich.pretty import Pretty
+from rich.prompt import Confirm
+
+from bfabric import Bfabric
+from bfabric.utils.cli_integration import use_client
+
+app = cyclopts.App()
+
+
+@cyclopts.Parameter(name="*")
+class Params(BaseModel):
+    endpoint: str
+    """The endpoint to delete from, e.g. 'resource'."""
+
+    id: list[int]
+    """The id or ids to delete."""
+
+    no_confirm: bool = False
+    """Whether to ask for confirmation before deleting."""
+
+
+def _perform_delete(client: Bfabric, endpoint: str, id: list[int]) -> None:
+    """Deletes the entity with the given id from the given endpoint."""
+    result = client.delete(endpoint=endpoint, id=id).to_list_dict()
+    logger.info(f"Entity with ID(s) {id} deleted successfully.")
+
+
+@app.default
+@logger.catch(reraise=True)
+@use_client
+def cli_api_delete(params: Params, *, client: Bfabric) -> None:
+    """Deletes entities from B-Fabric by id."""
+    if params.no_confirm:
+        _perform_delete(client=client, endpoint=params.endpoint, id=params.id)
+    else:
+        existing_entities = client.read(params.endpoint, {"id": params.id})
+
+        for id in params.id:
+            existing_entity = next((e for e in existing_entities if e["id"] == id), None)
+            if not existing_entity:
+                logger.warning(f"No entity found with ID {id}")
+                continue
+
+            rich.print(
+                Panel.fit(
+                    Pretty(existing_entity, expand_all=False), title=f"{params.endpoint.capitalize()} with ID {id}"
+                )
+            )
+            if Confirm.ask(f"Delete entity with ID {id}?"):
+                _perform_delete(client=client, endpoint=params.endpoint, id=[id])
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
index 9dc0cc46..45030329 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
@@ -4,9 +4,14 @@
 from bfabric_scripts.cli.api.cli_api_read import app as _cmd_read
 from bfabric_scripts.cli.api.cli_api_save import app as _cmd_save
 from bfabric_scripts.cli.api.cli_api_update import app as _cmd_update
+from bfabric_scripts.cli.api.cli_api_delete import app as _cmd_delete
 
 app = cyclopts.App()
-app.command(_cmd_log, name="log")
 app.command(_cmd_read, name="read")
 app.command(_cmd_update, name="update")
+app.command(_cmd_delete, name="delete")
+
+# TODO delete or move
+app.command(_cmd_log, name="log")
+# TODO delete
 app.command(_cmd_save, name="save")

From 91384c32d895f046a2d46da2225ba06caf089da2 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 08:52:01 +0100
Subject: [PATCH 04/35] mark deprecated

---
 bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py  | 2 +-
 bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_save.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py
index 3431db51..3ce3f20a 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py
@@ -7,7 +7,7 @@
 from bfabric import Bfabric
 from bfabric.utils.cli_integration import use_client
 
-cmd = cyclopts.App(help="write log messages of external jobs")
+cmd = cyclopts.App(help="DEPRECATED")
 
 log_target = cyclopts.Group(
     "Log Target",
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_save.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_save.py
index eff49bf9..3ae859b4 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_save.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_save.py
@@ -5,7 +5,7 @@
 from bfabric import Bfabric
 from bfabric.utils.cli_integration import use_client
 
-app = cyclopts.App()
+app = cyclopts.App(help="DEPRECATED")
 
 
 @app.default

From dd03c638b7fc8c7b5e5c60476158a493b3680ecd Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 09:04:58 +0100
Subject: [PATCH 05/35] add bfabric-cli api create

---
 bfabric_scripts/doc/changelog.md              |  1 +
 .../bfabric_scripts/cli/api/cli_api_create.py | 37 +++++++++++++++++++
 .../src/bfabric_scripts/cli/cli_api.py        |  6 ++-
 3 files changed, 42 insertions(+), 2 deletions(-)
 create mode 100644 bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py

diff --git a/bfabric_scripts/doc/changelog.md b/bfabric_scripts/doc/changelog.md
index 87d58b7b..a4f2655d 100644
--- a/bfabric_scripts/doc/changelog.md
+++ b/bfabric_scripts/doc/changelog.md
@@ -19,6 +19,7 @@ Versioning currently follows `X.Y.Z` where
 ### Added
 
 - `bfabric-cli api update` command to update an existing entity
+- `bfabric-cli api create` command to create a new entity
 - `bfabric-cli api delete` command to delete an existing entity
 
 ## \[1.13.20\] - 2025-02-10
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py
new file mode 100644
index 00000000..074e9e3b
--- /dev/null
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py
@@ -0,0 +1,37 @@
+import cyclopts
+from cyclopts import Parameter
+from loguru import logger
+from pydantic import BaseModel, field_validator, Field, ValidationError
+from rich.pretty import pprint
+
+from bfabric import Bfabric
+from bfabric.utils.cli_integration import use_client
+
+app = cyclopts.App()
+
+
+@Parameter(name="*")
+class Params(BaseModel):
+    endpoint: str
+    """Endpoint to update, e.g. 'resource'."""
+    attributes: list[tuple[str, str]] | None = Field(min_length=1)
+    """List of attribute-value pairs to update the entity with."""
+
+    @field_validator("attributes")
+    def _must_not_contain_id(cls, value):
+        if value:
+            for attribute, _ in value:
+                if attribute == "id":
+                    raise ValueError("Attribute 'id' is not allowed in the attributes.")
+        return value
+
+
+@app.default
+@use_client
+@logger.catch(reraise=True)
+def create(params: Params, *, client: Bfabric) -> None:
+    """Creates a new entity in B-Fabric."""
+    attributes_dict = {attribute: value for attribute, value in params.attributes}
+    result = client.save(params.endpoint, attributes_dict)
+    logger.info(f"{params.endpoint} entity with ID {result[0]["id"]} created successfully.")
+    pprint(result)
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
index 45030329..b77ad3bb 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
@@ -1,15 +1,17 @@
 import cyclopts
 
+from bfabric_scripts.cli.api.cli_api_create import app as _cmd_create
+from bfabric_scripts.cli.api.cli_api_delete import app as _cmd_delete
 from bfabric_scripts.cli.api.cli_api_log import cmd as _cmd_log
 from bfabric_scripts.cli.api.cli_api_read import app as _cmd_read
 from bfabric_scripts.cli.api.cli_api_save import app as _cmd_save
 from bfabric_scripts.cli.api.cli_api_update import app as _cmd_update
-from bfabric_scripts.cli.api.cli_api_delete import app as _cmd_delete
 
 app = cyclopts.App()
+app.command(_cmd_create, name="create")
+app.command(_cmd_delete, name="delete")
 app.command(_cmd_read, name="read")
 app.command(_cmd_update, name="update")
-app.command(_cmd_delete, name="delete")
 
 # TODO delete or move
 app.command(_cmd_log, name="log")

From d15a49458b01875c5e1cec5136417deca0c1a40c Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 09:05:29 +0100
Subject: [PATCH 06/35] better help

---
 bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py
index 65af0172..fdb28d23 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py
@@ -103,7 +103,7 @@ def render_output(results: list[dict[str, Any]], params: Params, client: Bfabric
 @use_client
 @logger.catch(reraise=True)
 def read(params: Annotated[Params, cyclopts.Parameter(name="*")], *, client: Bfabric) -> None | int:
-    """Reads one type of entity from B-Fabric."""
+    """Reads entities from B-Fabric."""
     console_user = Console(stderr=True)
     console_user.print(params)
 

From 7e7c880ded978bff8190f92875d9cc9f017dcb53 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 09:10:28 +0100
Subject: [PATCH 07/35] harmonize the executable subcommand

---
 bfabric_scripts/src/bfabric_scripts/cli/cli_executable.py | 8 ++++----
 .../cli/executable/{inspect.py => show.py}                | 3 ++-
 .../src/bfabric_scripts/cli/executable/upload.py          | 2 +-
 3 files changed, 7 insertions(+), 6 deletions(-)
 rename bfabric_scripts/src/bfabric_scripts/cli/executable/{inspect.py => show.py} (88%)

diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_executable.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_executable.py
index 1b9e96b9..ff24ea21 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/cli_executable.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_executable.py
@@ -1,8 +1,8 @@
 import cyclopts
 
-from bfabric_scripts.cli.executable.inspect import inspect_executable
-from bfabric_scripts.cli.executable.upload import upload_executable
+from bfabric_scripts.cli.executable.show import cmd_executable_show
+from bfabric_scripts.cli.executable.upload import cmd_executable_upload
 
 app = cyclopts.App()
-app.command(inspect_executable, name="inspect")
-app.command(upload_executable, name="upload")
+app.command(cmd_executable_show, name="show")
+app.command(cmd_executable_upload, name="upload")
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/executable/inspect.py b/bfabric_scripts/src/bfabric_scripts/cli/executable/show.py
similarity index 88%
rename from bfabric_scripts/src/bfabric_scripts/cli/executable/inspect.py
rename to bfabric_scripts/src/bfabric_scripts/cli/executable/show.py
index 521b6bef..d5ea7dd6 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/executable/inspect.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/executable/show.py
@@ -15,7 +15,8 @@ def get_storage_info(storage_id: int | None, client: Bfabric) -> dict[str, str |
 
 
 @use_client
-def inspect_executable(executable_id: int, *, client: Bfabric) -> None:
+def cmd_executable_show(executable_id: int, *, client: Bfabric) -> None:
+    """Show metadata and encoded program for an executable."""
     console = Console()
     executable = Executable.find(executable_id, client=client)
     metadata = {key: executable.get(key) for key in ("id", "name", "description", "relativepath", "context", "program")}
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/executable/upload.py b/bfabric_scripts/src/bfabric_scripts/cli/executable/upload.py
index d2c4fe0d..0f26fb24 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/executable/upload.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/executable/upload.py
@@ -10,7 +10,7 @@
 
 
 @use_client
-def upload_executable(executable_yaml: Path, *, upload: Path | None = None, client: Bfabric) -> None:
+def cmd_executable_upload(executable_yaml: Path, *, upload: Path | None = None, client: Bfabric) -> None:
     """Uploads an executable defined in the specified YAML to bfabric.
 
     :param executable_yaml: Path to the YAML file containing the executable data.

From 29de0fa6850064998d5af2b6069d883f480944be Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 09:11:15 +0100
Subject: [PATCH 08/35] fix Python 3.11 support

---
 bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py
index 074e9e3b..d7446e3a 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py
@@ -33,5 +33,5 @@ def create(params: Params, *, client: Bfabric) -> None:
     """Creates a new entity in B-Fabric."""
     attributes_dict = {attribute: value for attribute, value in params.attributes}
     result = client.save(params.endpoint, attributes_dict)
-    logger.info(f"{params.endpoint} entity with ID {result[0]["id"]} created successfully.")
+    logger.info(f"{params.endpoint} entity with ID {result[0]['id']} created successfully.")
     pprint(result)

From 6cab8d881f95017dda7bff9760d0df34e689badb Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 14:41:39 +0100
Subject: [PATCH 09/35] add Makefile for easier app-runner interaction

---
 bfabric_app_runner/docs/changelog.md          |  1 +
 .../src/bfabric_app_runner/cli/app.py         | 20 +++++
 .../bfabric_app_runner/resources/workunit.mk  | 87 +++++++++++++++++++
 3 files changed, 108 insertions(+)
 create mode 100644 bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk

diff --git a/bfabric_app_runner/docs/changelog.md b/bfabric_app_runner/docs/changelog.md
index 87200f1b..46d9661c 100644
--- a/bfabric_app_runner/docs/changelog.md
+++ b/bfabric_app_runner/docs/changelog.md
@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ### Added
 
 - Implement `--force-storage` to pass a yaml to a forced storage instead of the real one.
+- A Makefile will be created in the app folder for easier interaction with the app-runner (it uses uv and PyPI).
 
 ### Changed
 
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
index 313e6ce0..3064deb2 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
@@ -1,8 +1,10 @@
 from __future__ import annotations
 
+import importlib.metadata
 from pathlib import Path
 
 import cyclopts
+from loguru import logger
 
 from bfabric_app_runner.app_runner.resolve_app import load_workunit_information
 from bfabric_app_runner.app_runner.runner import run_app, Runner
@@ -28,6 +30,8 @@ def run(
     client = Bfabric.from_config()
     app_version, workunit_ref = load_workunit_information(app_spec, client, work_dir, workunit_ref)
 
+    copy_dev_makefile(work_dir=work_dir)
+
     # TODO(#107): usage of entity lookup cache was problematic -> beyond the full solution we could also consider
     #             to deactivate the cache for the output registration
     # with EntityLookupCache.enable():
@@ -63,3 +67,19 @@ def dispatch(
     with EntityLookupCache.enable():
         runner = Runner(spec=app_version, client=client, ssh_user=None)
         runner.run_dispatch(workunit_ref=workunit_ref, work_dir=work_dir)
+
+
+def copy_dev_makefile(work_dir: Path) -> None:
+    """Copies the workunit.mk file to the work directory, and sets the version of the app runner."""
+    source_path = Path(__file__).parents[1] / "resources" / "workunit.mk"
+    target_path = work_dir / "Makefile"
+
+    makefile_template = target_path.read_text()
+    app_runner_version = importlib.metadata.version("bfabric_app_runner")
+    makefile = makefile_template.replace("@RUNNER_VERSION@", app_runner_version)
+
+    if target_path.exists():
+        logger.info("Renaming existing Makefile to Makefile.bak")
+        target_path.rename(work_dir / "Makefile.bak")
+    logger.info(f"Copying Makefile from {source_path} to {target_path}")
+    target_path.write_text(makefile)
diff --git a/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk b/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk
new file mode 100644
index 00000000..559f4476
--- /dev/null
+++ b/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk
@@ -0,0 +1,87 @@
+# Makefile for bfabric-app-runner operations
+#
+# Quick Start Guide:
+# -----------------
+# For most cases, just run:
+#     make run-all
+#
+# For step-by-step execution:
+# 1. make dispatch              # Creates chunks.yml
+# 2. make inputs WORK_DIR=dir   # Prepares input files
+# 3. make process WORK_DIR=dir  # Processes the chunks in specified directory
+# 4. make stage WORK_DIR=dir    # Stages results to server/storage
+#
+# The WORK_DIR parameter defaults to "work" if not specified.
+# Example with custom directory:
+#     make inputs WORK_DIR=work_folder_1
+#     make process WORK_DIR=work_folder_1
+#     make stage WORK_DIR=work_folder_1
+#
+# Use `make help` to see all available commands
+
+# Configuration
+RUNNER_VERSION := @RUNNER_VERSION@
+RUNNER_CMD := uv run --with bfabric-app-runner==$(RUNNER_VERSION) bfabric-app-runner
+
+# Input files
+APP_DEF := app_definition.yml
+WORKUNIT_DEF := $(realpath workunit_definition.yml)
+
+# Default work directory (can be overridden via command line)
+WORK_DIR ?= work
+
+.PHONY: help dispatch inputs process stage run-all clean
+
+# Default target
+help:
+	@echo "Available commands:"
+	@echo "  make run-all                 - Run all steps in a single command (recommended for most cases)"
+	@echo ""
+	@echo "Step-by-step execution:"
+	@echo "  make dispatch                - Step 1: Initial step (creates chunks.yml)"
+	@echo "  make inputs [WORK_DIR=dir]   - Step 2: Prepare input files"
+	@echo "  make process [WORK_DIR=dir]  - Step 3: Process chunks in specified directory"
+	@echo "  make stage [WORK_DIR=dir]    - Step 4: Stage results to server/storage"
+	@echo ""
+	@echo "Other commands:"
+	@echo "  make clean [WORK_DIR=dir]    - Remove specified work directory"
+	@echo "  make help                    - Show this help message"
+	@echo ""
+	@echo "Current settings:"
+	@echo "  WORK_DIR = $(WORK_DIR) (default: work)"
+
+# Step 1: Initial dispatch
+dispatch:
+	@echo "Step 1/4: Running initial dispatch..."
+	$(RUNNER_CMD) app dispatch $(APP_DEF) $(PWD) $(WORKUNIT_DEF)
+	@echo "✓ Dispatch completed - chunks.yml created"
+
+# Step 2: Prepare inputs
+inputs:
+	@echo "Step 2/4: Preparing inputs in directory '$(WORK_DIR)'..."
+	$(RUNNER_CMD) inputs prepare $(WORK_DIR)/inputs.yml
+	@echo "✓ Inputs prepared for '$(WORK_DIR)'"
+
+# Step 3: Process chunks
+process:
+	@echo "Step 3/4: Processing chunks in directory '$(WORK_DIR)'..."
+	$(RUNNER_CMD) chunk process $(APP_DEF) $(WORK_DIR)
+	@echo "✓ Processing completed for '$(WORK_DIR)'"
+
+# Step 4: Stage results
+stage:
+	@echo "Step 4/4: Staging results from directory '$(WORK_DIR)'..."
+	$(RUNNER_CMD) chunk outputs $(APP_DEF) $(WORK_DIR) $(WORKUNIT_DEF)
+	@echo "✓ Results staged for '$(WORK_DIR)'"
+
+# Run all steps in one command
+run-all:
+	@echo "Running all steps in a single command..."
+	$(RUNNER_CMD) app run $(APP_DEF) . $(WORKUNIT_DEF)
+	@echo "✓ All steps completed"
+
+# Clean generated files
+clean:
+	@echo "Cleaning directory '$(WORK_DIR)'..."
+	rm -rf $(WORK_DIR)
+	@echo "✓ Clean completed for '$(WORK_DIR)'"

From 440ef8f11c4a4c49c752de19c7925f34e39ccfef Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 14:44:35 +0100
Subject: [PATCH 10/35] make the Makefile copy work

---
 bfabric_app_runner/pyproject.toml             |  7 ++++
 .../src/bfabric_app_runner/cli/app.py         | 32 ++++++++++---------
 2 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/bfabric_app_runner/pyproject.toml b/bfabric_app_runner/pyproject.toml
index a7860fdd..5e97e24b 100644
--- a/bfabric_app_runner/pyproject.toml
+++ b/bfabric_app_runner/pyproject.toml
@@ -18,6 +18,13 @@ dependencies = [
     "mako",
 ]
 
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/bfabric_app_runner/**/*.py",
+    "src/bfabric_app_runner/py.typed",
+    "src/bfabric_app_runner/resources/*.mk"
+]
+
 [project.scripts]
 "bfabric-app-runner"="bfabric_app_runner.cli.__main__:app"
 
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
index 3064deb2..e2f741a6 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
@@ -1,16 +1,17 @@
 from __future__ import annotations
 
 import importlib.metadata
+import importlib.resources
 from pathlib import Path
 
 import cyclopts
 from loguru import logger
 
-from bfabric_app_runner.app_runner.resolve_app import load_workunit_information
-from bfabric_app_runner.app_runner.runner import run_app, Runner
 from bfabric import Bfabric
-from bfabric.utils.cli_integration import setup_script_logging
 from bfabric.experimental.entity_lookup_cache import EntityLookupCache
+from bfabric.utils.cli_integration import setup_script_logging
+from bfabric_app_runner.app_runner.resolve_app import load_workunit_information
+from bfabric_app_runner.app_runner.runner import run_app, Runner
 
 app_app = cyclopts.App("app", help="Run an app.")
 
@@ -71,15 +72,16 @@ def dispatch(
 
 def copy_dev_makefile(work_dir: Path) -> None:
     """Copies the workunit.mk file to the work directory, and sets the version of the app runner."""
-    source_path = Path(__file__).parents[1] / "resources" / "workunit.mk"
-    target_path = work_dir / "Makefile"
-
-    makefile_template = target_path.read_text()
-    app_runner_version = importlib.metadata.version("bfabric_app_runner")
-    makefile = makefile_template.replace("@RUNNER_VERSION@", app_runner_version)
-
-    if target_path.exists():
-        logger.info("Renaming existing Makefile to Makefile.bak")
-        target_path.rename(work_dir / "Makefile.bak")
-    logger.info(f"Copying Makefile from {source_path} to {target_path}")
-    target_path.write_text(makefile)
+    with importlib.resources.path("bfabric_app_runner", "resources/workunit.mk") as source_path:
+        target_path = work_dir / "Makefile"
+
+        makefile_template = source_path.read_text()
+        app_runner_version = importlib.metadata.version("bfabric_app_runner")
+        makefile = makefile_template.replace("@RUNNER_VERSION@", app_runner_version)
+
+        if target_path.exists():
+            logger.info("Renaming existing Makefile to Makefile.bak")
+            target_path.rename(work_dir / "Makefile.bak")
+        logger.info(f"Copying Makefile from {source_path} to {target_path}")
+        target_path.parent.mkdir(exist_ok=True, parents=True)
+        target_path.write_text(makefile)

From 370dfb45d6d9ee2efb7b8139bc15a0fcd3a3782b Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 15:08:48 +0100
Subject: [PATCH 11/35] safer makefile

---
 .../bfabric_app_runner/resources/workunit.mk    | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk b/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk
index 559f4476..6b92bfaa 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk
+++ b/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk
@@ -21,11 +21,12 @@
 
 # Configuration
 RUNNER_VERSION := @RUNNER_VERSION@
-RUNNER_CMD := uv run --with bfabric-app-runner==$(RUNNER_VERSION) bfabric-app-runner
+RUNNER_CMD := uv run --with "bfabric-app-runner==$(RUNNER_VERSION)" bfabric-app-runner
 
 # Input files
-APP_DEF := app_definition.yml
+APP_DEF := $(realpath app_definition.yml)
 WORKUNIT_DEF := $(realpath workunit_definition.yml)
+CURRENT_DIR := $(shell pwd)
 
 # Default work directory (can be overridden via command line)
 WORK_DIR ?= work
@@ -53,35 +54,35 @@ help:
 # Step 1: Initial dispatch
 dispatch:
 	@echo "Step 1/4: Running initial dispatch..."
-	$(RUNNER_CMD) app dispatch $(APP_DEF) $(PWD) $(WORKUNIT_DEF)
+	$(RUNNER_CMD) app dispatch "$(APP_DEF)" "$(CURRENT_DIR)" "$(WORKUNIT_DEF)"
 	@echo "✓ Dispatch completed - chunks.yml created"
 
 # Step 2: Prepare inputs
 inputs:
 	@echo "Step 2/4: Preparing inputs in directory '$(WORK_DIR)'..."
-	$(RUNNER_CMD) inputs prepare $(WORK_DIR)/inputs.yml
+	$(RUNNER_CMD) inputs prepare "$(WORK_DIR)/inputs.yml"
 	@echo "✓ Inputs prepared for '$(WORK_DIR)'"
 
 # Step 3: Process chunks
 process:
 	@echo "Step 3/4: Processing chunks in directory '$(WORK_DIR)'..."
-	$(RUNNER_CMD) chunk process $(APP_DEF) $(WORK_DIR)
+	$(RUNNER_CMD) chunk process "$(APP_DEF)" "$(WORK_DIR)"
 	@echo "✓ Processing completed for '$(WORK_DIR)'"
 
 # Step 4: Stage results
 stage:
 	@echo "Step 4/4: Staging results from directory '$(WORK_DIR)'..."
-	$(RUNNER_CMD) chunk outputs $(APP_DEF) $(WORK_DIR) $(WORKUNIT_DEF)
+	$(RUNNER_CMD) chunk outputs "$(APP_DEF)" "$(WORK_DIR)" "$(WORKUNIT_DEF)"
 	@echo "✓ Results staged for '$(WORK_DIR)'"
 
 # Run all steps in one command
 run-all:
 	@echo "Running all steps in a single command..."
-	$(RUNNER_CMD) app run $(APP_DEF) . $(WORKUNIT_DEF)
+	$(RUNNER_CMD) app run "$(APP_DEF)" "." "$(WORKUNIT_DEF)"
 	@echo "✓ All steps completed"
 
 # Clean generated files
 clean:
 	@echo "Cleaning directory '$(WORK_DIR)'..."
-	rm -rf $(WORK_DIR)
+	rm -rf "$(WORK_DIR)"
 	@echo "✓ Clean completed for '$(WORK_DIR)'"

From 8bf6e17186f5b58e2b8b27ee8d3ad64db2b4e705 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 15:16:36 +0100
Subject: [PATCH 12/35] remove pytest.ini which probably had no effect

---
 ___pytest.ini | 2 --
 1 file changed, 2 deletions(-)
 delete mode 100644 ___pytest.ini

diff --git a/___pytest.ini b/___pytest.ini
deleted file mode 100644
index 23544c67..00000000
--- a/___pytest.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[pytest]
-logot_capturer = logot.loguru.LoguruCapturer

From f4959592767c2bbc230b3b78f41764613b476bb9 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Thu, 13 Feb 2025 15:30:36 +0100
Subject: [PATCH 13/35] use the `app_version.yml` file now

---
 bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk b/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk
index 6b92bfaa..a03a58c4 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk
+++ b/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk
@@ -24,7 +24,7 @@ RUNNER_VERSION := @RUNNER_VERSION@
 RUNNER_CMD := uv run --with "bfabric-app-runner==$(RUNNER_VERSION)" bfabric-app-runner
 
 # Input files
-APP_DEF := $(realpath app_definition.yml)
+APP_DEF := $(realpath app_version.yml)
 WORKUNIT_DEF := $(realpath workunit_definition.yml)
 CURRENT_DIR := $(shell pwd)
 

From 2e616e204d777ff8588115904ebcfa38f7d2fc29 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Mon, 17 Feb 2025 10:43:48 +0100
Subject: [PATCH 14/35] implement webapp token support

---
 bfabric/pyproject.toml                 |  1 +
 bfabric/src/bfabric/bfabric.py         | 28 +++++++++++++--
 bfabric/src/bfabric/rest/__init__.py   |  0
 bfabric/src/bfabric/rest/token_data.py | 49 ++++++++++++++++++++++++++
 docs/changelog.md                      |  2 ++
 5 files changed, 77 insertions(+), 3 deletions(-)
 create mode 100644 bfabric/src/bfabric/rest/__init__.py
 create mode 100644 bfabric/src/bfabric/rest/token_data.py

diff --git a/bfabric/pyproject.toml b/bfabric/pyproject.toml
index 08827c33..df26c2f5 100644
--- a/bfabric/pyproject.toml
+++ b/bfabric/pyproject.toml
@@ -29,6 +29,7 @@ dependencies = [
     "eval_type_backport; python_version < '3.10'",
     "python-dateutil >= 2.9.0",
     "cyclopts >= 2.9.9",
+    "requests >= 2.26.0",
     #"platformdirs >= 4.3",
 ]
 
diff --git a/bfabric/src/bfabric/bfabric.py b/bfabric/src/bfabric/bfabric.py
index eeb140b5..dc30728f 100644
--- a/bfabric/src/bfabric/bfabric.py
+++ b/bfabric/src/bfabric/bfabric.py
@@ -15,7 +15,6 @@
 
 import base64
 import importlib.metadata
-import sys
 from contextlib import contextmanager
 from datetime import datetime
 from enum import Enum
@@ -24,16 +23,18 @@
 from pprint import pprint
 from typing import Literal, Any, TYPE_CHECKING
 
+import sys
 from loguru import logger
 from rich.console import Console
 
 from bfabric.bfabric_config import read_config
-from bfabric.utils.cli_integration import DEFAULT_THEME, HostnameHighlighter
 from bfabric.config import BfabricAuth
 from bfabric.config import BfabricClientConfig
 from bfabric.engine.engine_suds import EngineSUDS
 from bfabric.engine.engine_zeep import EngineZeep
+from bfabric.rest.token_data import get_token_data
 from bfabric.results.result_container import ResultContainer
+from bfabric.utils.cli_integration import DEFAULT_THEME, HostnameHighlighter
 from bfabric.utils.paginator import compute_requested_pages, BFABRIC_QUERY_LIMIT
 
 if TYPE_CHECKING:
@@ -93,12 +94,33 @@ def from_config(
         :param config_path: Path to the config file, in case it is different from default
         :param auth: Authentication to use. If "config" is given, the authentication will be read from the config file.
             If it is set to None, no authentication will be used.
-        :param engine: Engine to use for the API. Default is SUDS.
+        :param engine: Engine to use for the API.
         """
         config, auth_config = get_system_auth(config_env=config_env, config_path=config_path)
         auth_used: BfabricAuth | None = auth_config if auth == "config" else auth
         return cls(config, auth_used, engine=engine)
 
+    @classmethod
+    def from_token(
+        cls,
+        token: str,
+        config_env: str | None = None,
+        config_path: str | None = None,
+        engine: BfabricAPIEngineType = BfabricAPIEngineType.SUDS,
+    ) -> Bfabric:
+        """Returns a new Bfabric instance, configured with the user configuration file and the provided token.
+        Any authentication in the configuration file will be ignored, but it will be used to determine the correct
+        B-Fabric instance.
+        :param token: the token to use for authentication
+        :param config_env: the config environment to use (if not specified, see `from_config`)
+        :param config_path: the path to the config file (if not specified, see `from_config`)
+        :param engine: the engine to use for the API.
+        """
+        config, _ = get_system_auth(config_env=config_env, config_path=config_path)
+        token_data = get_token_data(client_config=config, token=token)
+        auth = BfabricAuth(login=token_data.user, password=token_data.user_ws_password.get_secret_value())
+        return cls(config, auth, engine=engine)
+
     @property
     def config(self) -> BfabricClientConfig:
         """Returns the config object."""
diff --git a/bfabric/src/bfabric/rest/__init__.py b/bfabric/src/bfabric/rest/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/bfabric/src/bfabric/rest/token_data.py b/bfabric/src/bfabric/rest/token_data.py
new file mode 100644
index 00000000..8024f760
--- /dev/null
+++ b/bfabric/src/bfabric/rest/token_data.py
@@ -0,0 +1,49 @@
+from __future__ import annotations
+from datetime import datetime
+from enum import Enum
+from typing import TYPE_CHECKING
+
+import requests
+from pydantic import BaseModel, Field, SecretStr
+
+if TYPE_CHECKING:
+    from bfabric import BfabricClientConfig
+
+
+class Environment(Enum):
+    Test = "Test"
+    Production = "Production"
+
+
+class TokenData(BaseModel):
+    """Parsed token data from the B-Fabric token validation endpoint."""
+
+    job_id: int = Field(alias="jobId")
+    application_id: int = Field(alias="applicationId")
+
+    entity_class: str = Field(alias="entityClassName")
+    entity_id: int = Field(alias="entityId")
+
+    user: str = Field(alias="user")
+    user_ws_password: SecretStr = Field(alias="userWsPassword")
+
+    token_expires: datetime = Field(alias="expiryDateTime")
+    environment: Environment
+
+    class Config:
+        populate_by_name = True
+        str_strip_whitespace = True
+        json_encoders = {datetime: lambda v: v.isoformat()}
+
+
+def get_token_data(client_config: BfabricClientConfig, token: str) -> TokenData:
+    """Returns the token data for the provided token.
+
+    If the request fails, an exception is raised.
+    """
+    url = f"{client_config.base_url}/rest/token/validate"
+    response = requests.get(url, params={"token": token})
+    if not response.ok:
+        response.raise_for_status()
+    parsed = response.json()
+    return TokenData.model_validate(parsed)
diff --git a/docs/changelog.md b/docs/changelog.md
index 25d5bc1c..bd8a01d1 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -13,6 +13,8 @@ Versioning currently follows `X.Y.Z` where
 ### Added
 
 - `Entity.load_yaml` and `Entity.dump_yaml`
+- `Bfabric.from_token` to create a `Bfabric` instance from a token
+- `bfabric.rest.token_data` to get token data from the REST API, low-level functionality
 
 ## \[1.13.20\] - 2025-02-10
 

From 5c6f2c44aa23b5ba88d5b9e9a746d069da8bda28 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Mon, 17 Feb 2025 10:47:05 +0100
Subject: [PATCH 15/35] dont parse it for better compatibility

---
 bfabric/src/bfabric/rest/token_data.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/bfabric/src/bfabric/rest/token_data.py b/bfabric/src/bfabric/rest/token_data.py
index 8024f760..13172ea7 100644
--- a/bfabric/src/bfabric/rest/token_data.py
+++ b/bfabric/src/bfabric/rest/token_data.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 from datetime import datetime
-from enum import Enum
 from typing import TYPE_CHECKING
 
 import requests
@@ -10,11 +9,6 @@
     from bfabric import BfabricClientConfig
 
 
-class Environment(Enum):
-    Test = "Test"
-    Production = "Production"
-
-
 class TokenData(BaseModel):
     """Parsed token data from the B-Fabric token validation endpoint."""
 
@@ -28,7 +22,7 @@ class TokenData(BaseModel):
     user_ws_password: SecretStr = Field(alias="userWsPassword")
 
     token_expires: datetime = Field(alias="expiryDateTime")
-    environment: Environment
+    environment: str
 
     class Config:
         populate_by_name = True

From a8c277a47a6f5843ff9f24ef1fdb8b3e81d33b37 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Mon, 17 Feb 2025 12:36:30 +0100
Subject: [PATCH 16/35] refactor: update password handling to use SecretStr

---
 bfabric/src/bfabric/bfabric.py                          | 2 +-
 bfabric/src/bfabric/config/bfabric_auth.py              | 9 ++-------
 bfabric/src/bfabric/engine/engine_suds.py               | 6 +++---
 bfabric/src/bfabric/engine/engine_zeep.py               | 6 +++---
 bfabric/src/bfabric/examples/compare_zeep_suds_query.py | 2 +-
 bfabric/src/bfabric/examples/zeep_debug.py              | 2 +-
 docs/changelog.md                                       | 4 ++++
 tests/bfabric/config/test_bfabric_auth.py               | 7 +++++--
 tests/bfabric/config/test_bfabric_client_config.py      | 4 ++--
 tests/bfabric/config/test_config_file.py                | 6 +++---
 tests/bfabric/engine/test_engine_suds.py                | 3 ++-
 tests/bfabric/engine/test_engine_zeep.py                | 3 ++-
 tests/bfabric/test_bfabric_config.py                    | 2 +-
 13 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/bfabric/src/bfabric/bfabric.py b/bfabric/src/bfabric/bfabric.py
index dc30728f..8d13a20d 100644
--- a/bfabric/src/bfabric/bfabric.py
+++ b/bfabric/src/bfabric/bfabric.py
@@ -118,7 +118,7 @@ def from_token(
         """
         config, _ = get_system_auth(config_env=config_env, config_path=config_path)
         token_data = get_token_data(client_config=config, token=token)
-        auth = BfabricAuth(login=token_data.user, password=token_data.user_ws_password.get_secret_value())
+        auth = BfabricAuth(login=token_data.user, password=token_data.user_ws_password)
         return cls(config, auth, engine=engine)
 
     @property
diff --git a/bfabric/src/bfabric/config/bfabric_auth.py b/bfabric/src/bfabric/config/bfabric_auth.py
index 43b8a7e9..84b2dcdf 100644
--- a/bfabric/src/bfabric/config/bfabric_auth.py
+++ b/bfabric/src/bfabric/config/bfabric_auth.py
@@ -2,16 +2,11 @@
 
 from typing import Annotated
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, SecretStr
 
 
 class BfabricAuth(BaseModel):
     """Holds the authentication data for the B-Fabric client."""
 
     login: Annotated[str, Field(min_length=3)]
-    password: Annotated[str, Field(min_length=32, max_length=32)]
-
-    def __repr__(self) -> str:
-        return f"BfabricAuth(login={repr(self.login)}, password=...)"
-
-    __str__ = __repr__
+    password: Annotated[SecretStr, Field(min_length=32, max_length=32)]
diff --git a/bfabric/src/bfabric/engine/engine_suds.py b/bfabric/src/bfabric/engine/engine_suds.py
index 592d7c6f..54e149f8 100644
--- a/bfabric/src/bfabric/engine/engine_suds.py
+++ b/bfabric/src/bfabric/engine/engine_suds.py
@@ -49,7 +49,7 @@ def read(
         full_query = dict(
             login=auth.login,
             page=page,
-            password=auth.password,
+            password=auth.password.get_secret_value(),
             query=query,
             idonly=return_id_only,
         )
@@ -65,7 +65,7 @@ def save(self, endpoint: str, obj: dict, auth: BfabricAuth, method: str = "save"
         :param method: the method to use for saving, generally "save", but in some cases e.g. "checkandinsert" is more
             appropriate to be used instead.
         """
-        query = {"login": auth.login, "password": auth.password, endpoint: obj}
+        query = {"login": auth.login, "password": auth.password.get_secret_value(), endpoint: obj}
         service = self._get_suds_service(endpoint)
         try:
             response = getattr(service, method)(query)
@@ -84,7 +84,7 @@ def delete(self, endpoint: str, id: int | list[int], auth: BfabricAuth) -> Resul
             # TODO maybe use error here (and make sure it's consistent)
             return ResultContainer([], total_pages_api=0)
 
-        query = {"login": auth.login, "password": auth.password, "id": id}
+        query = {"login": auth.login, "password": auth.password.get_secret_value(), "id": id}
         service = self._get_suds_service(endpoint)
         response = service.delete(query)
         return self._convert_results(response=response, endpoint=endpoint)
diff --git a/bfabric/src/bfabric/engine/engine_zeep.py b/bfabric/src/bfabric/engine/engine_zeep.py
index 75f883ef..51c4adaf 100644
--- a/bfabric/src/bfabric/engine/engine_zeep.py
+++ b/bfabric/src/bfabric/engine/engine_zeep.py
@@ -59,7 +59,7 @@ def read(
         full_query = dict(
             login=auth.login,
             page=page,
-            password=auth.password,
+            password=auth.password.get_secret_value(),
             query=query,
             idonly=return_id_only,
         )
@@ -84,7 +84,7 @@ def save(self, endpoint: str, obj: dict, auth: BfabricAuth, method: str = "save"
             excl_keys = ["name", "sampleid", "storageid", "workunitid", "relativepath"]
             _zeep_query_append_skipped(query, excl_keys, inplace=True, overwrite=False)
 
-        full_query = {"login": auth.login, "password": auth.password, endpoint: query}
+        full_query = {"login": auth.login, "password": auth.password.get_secret_value(), endpoint: query}
 
         client = self._get_client(endpoint)
 
@@ -108,7 +108,7 @@ def delete(self, endpoint: str, id: int | list[int], auth: BfabricAuth) -> Resul
             # TODO maybe use error here (and make sure it's consistent)
             return ResultContainer([], total_pages_api=0)
 
-        query = {"login": auth.login, "password": auth.password, "id": id}
+        query = {"login": auth.login, "password": auth.password.get_secret_value(), "id": id}
 
         client = self._get_client(endpoint)
         response = client.service.delete(query)
diff --git a/bfabric/src/bfabric/examples/compare_zeep_suds_query.py b/bfabric/src/bfabric/examples/compare_zeep_suds_query.py
index 04e00f75..8eeb4bfc 100644
--- a/bfabric/src/bfabric/examples/compare_zeep_suds_query.py
+++ b/bfabric/src/bfabric/examples/compare_zeep_suds_query.py
@@ -52,7 +52,7 @@ def full_query(auth: BfabricAuth, query: dict, includedeletableupdateable: bool
     thisQuery = deepcopy(query)
     thisQuery["includedeletableupdateable"] = includedeletableupdateable
 
-    return {"login": auth.login, "password": auth.password, "query": thisQuery}
+    return {"login": auth.login, "password": auth.password.get_secret_value(), "query": thisQuery}
 
 
 def calc_both(
diff --git a/bfabric/src/bfabric/examples/zeep_debug.py b/bfabric/src/bfabric/examples/zeep_debug.py
index 15c7e287..7de918ab 100644
--- a/bfabric/src/bfabric/examples/zeep_debug.py
+++ b/bfabric/src/bfabric/examples/zeep_debug.py
@@ -22,7 +22,7 @@ def full_query(auth: BfabricAuth, query: dict, includedeletableupdateable: bool
     thisQuery = deepcopy(query)
     thisQuery["includedeletableupdateable"] = includedeletableupdateable
 
-    return {"login": auth.login, "password": auth.password, "query": thisQuery}
+    return {"login": auth.login, "password": auth.password.get_secret_value(), "query": thisQuery}
 
 
 def read_zeep(wsdl, fullQuery, raw=True):
diff --git a/docs/changelog.md b/docs/changelog.md
index bd8a01d1..6591aef8 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -16,6 +16,10 @@ Versioning currently follows `X.Y.Z` where
 - `Bfabric.from_token` to create a `Bfabric` instance from a token
 - `bfabric.rest.token_data` to get token data from the REST API, low-level functionality
 
+### Changed
+
+- Internally, the user password is now in a `pydantic.SecretStr` until we construct the API call. This should prevent some logging related accidents.
+
 ## \[1.13.20\] - 2025-02-10
 
 ### Breaking
diff --git a/tests/bfabric/config/test_bfabric_auth.py b/tests/bfabric/config/test_bfabric_auth.py
index 8cbeadca..a97fa10d 100644
--- a/tests/bfabric/config/test_bfabric_auth.py
+++ b/tests/bfabric/config/test_bfabric_auth.py
@@ -11,11 +11,14 @@ def example_config_path() -> Path:
 
 
 def test_bfabric_auth_repr() -> None:
-    assert repr(BfabricAuth(login="login", password="x" * 32)) == "BfabricAuth(login='login', password=...)"
+    assert (
+        repr(BfabricAuth(login="login", password="x" * 32))
+        == "BfabricAuth(login='login', password=SecretStr('**********'))"
+    )
 
 
 def test_bfabric_auth_str() -> None:
-    assert str(BfabricAuth(login="login", password="x" * 32)) == "BfabricAuth(login='login', password=...)"
+    assert str(BfabricAuth(login="login", password="x" * 32)) == "login='login' password=SecretStr('**********')"
 
 
 if __name__ == "__main__":
diff --git a/tests/bfabric/config/test_bfabric_client_config.py b/tests/bfabric/config/test_bfabric_client_config.py
index c156efc9..a226ab20 100644
--- a/tests/bfabric/config/test_bfabric_client_config.py
+++ b/tests/bfabric/config/test_bfabric_client_config.py
@@ -70,7 +70,7 @@ def test_bfabric_config_read_yml_bypath_default(mocker: MockerFixture, example_c
 
     config, auth = read_config(example_config_path)
     assert auth.login == "my_epic_production_login"
-    assert auth.password == "01234567890123456789012345678901"
+    assert auth.password.get_secret_value() == "01234567890123456789012345678901"
     assert config.base_url == "https://mega-production-server.uzh.ch/myprod"
 
     logot.assert_logged(logged.debug(f"Reading configuration from: {str(example_config_path.absolute())}"))
@@ -85,7 +85,7 @@ def test_bfabric_config_read_yml_bypath_environment_variable(
 
     config, auth = read_config(example_config_path)
     assert auth.login == "my_epic_test_login"
-    assert auth.password == "012345678901234567890123456789ff"
+    assert auth.password.get_secret_value() == "012345678901234567890123456789ff"
     assert config.base_url == "https://mega-test-server.uzh.ch/mytest"
 
     logot.assert_logged(logged.debug(f"Reading configuration from: {str(example_config_path.absolute())}"))
diff --git a/tests/bfabric/config/test_config_file.py b/tests/bfabric/config/test_config_file.py
index cab1129d..136f1753 100644
--- a/tests/bfabric/config/test_config_file.py
+++ b/tests/bfabric/config/test_config_file.py
@@ -54,7 +54,7 @@ def test_environment_config_when_auth(data_with_auth):
     config = EnvironmentConfig.model_validate(data_with_auth["PRODUCTION"])
     assert config.config.base_url == "https://example.com/"
     assert config.auth.login == "test-dummy"
-    assert config.auth.password == "00000000001111111111222222222233"
+    assert config.auth.password.get_secret_value() == "00000000001111111111222222222233"
 
 
 def test_environment_config_when_no_auth(data_no_auth):
@@ -69,7 +69,7 @@ def test_config_file_when_auth(data_with_auth):
     assert len(config.environments) == 1
     assert config.environments["PRODUCTION"].config.base_url == "https://example.com/"
     assert config.environments["PRODUCTION"].auth.login == "test-dummy"
-    assert config.environments["PRODUCTION"].auth.password == "00000000001111111111222222222233"
+    assert config.environments["PRODUCTION"].auth.password.get_secret_value() == "00000000001111111111222222222233"
 
 
 def test_config_file_when_no_auth(data_no_auth):
@@ -88,7 +88,7 @@ def test_config_file_when_multiple(data_multiple):
     assert config.environments["PRODUCTION"].auth is None
     assert config.environments["TEST"].config.base_url == "https://test.example.com/"
     assert config.environments["TEST"].auth.login == "test-dummy"
-    assert config.environments["TEST"].auth.password == "00000000001111111111222222222233"
+    assert config.environments["TEST"].auth.password.get_secret_value() == "00000000001111111111222222222233"
 
 
 def test_config_file_when_non_existent_default(data_no_auth):
diff --git a/tests/bfabric/engine/test_engine_suds.py b/tests/bfabric/engine/test_engine_suds.py
index 99631b3d..62d0e59d 100644
--- a/tests/bfabric/engine/test_engine_suds.py
+++ b/tests/bfabric/engine/test_engine_suds.py
@@ -1,6 +1,7 @@
 from unittest.mock import MagicMock
 
 import pytest
+from pydantic import SecretStr
 from suds import MethodNotFound
 from suds.client import Client
 
@@ -16,7 +17,7 @@ def engine_suds():
 
 @pytest.fixture
 def mock_auth():
-    return MagicMock(login="test_user", password="test_pass")
+    return MagicMock(login="test_user", password=SecretStr("test_pass"))
 
 
 @pytest.fixture
diff --git a/tests/bfabric/engine/test_engine_zeep.py b/tests/bfabric/engine/test_engine_zeep.py
index 956d7d85..3e8939d5 100644
--- a/tests/bfabric/engine/test_engine_zeep.py
+++ b/tests/bfabric/engine/test_engine_zeep.py
@@ -2,6 +2,7 @@
 
 import pytest
 import zeep
+from pydantic import SecretStr
 
 from bfabric.engine.engine_zeep import EngineZeep, _zeep_query_append_skipped
 from bfabric.errors import BfabricRequestError
@@ -15,7 +16,7 @@ def engine_zeep():
 
 @pytest.fixture
 def mock_auth():
-    return MagicMock(login="test_user", password="test_pass")
+    return MagicMock(login="test_user", password=SecretStr("test_pass"))
 
 
 @pytest.fixture
diff --git a/tests/bfabric/test_bfabric_config.py b/tests/bfabric/test_bfabric_config.py
index bb1ddd70..8a928368 100644
--- a/tests/bfabric/test_bfabric_config.py
+++ b/tests/bfabric/test_bfabric_config.py
@@ -22,7 +22,7 @@ def test_read_yml_bypath_all_fields(example_config_path: Path) -> None:
     job_notification_emails_ground_truth = "john.snow@fgcz.uzh.ch billy.the.kid@fgcz.ethz.ch"
 
     assert auth.login == "my_epic_test_login"
-    assert auth.password == "012345678901234567890123456789ff"
+    assert auth.password.get_secret_value() == "012345678901234567890123456789ff"
     assert config.base_url == "https://mega-test-server.uzh.ch/mytest"
     assert config.application_ids == applications_dict_ground_truth
     assert config.job_notification_emails == job_notification_emails_ground_truth

From 2dcf309733a67af2e85b1a458345d5f59311c9b4 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leoschwarz@users.noreply.github.com>
Date: Mon, 17 Feb 2025 15:26:40 +0100
Subject: [PATCH 17/35] Add `bfabric-cli dataset` (#145)

---
 bfabric/src/bfabric/entities/dataset.py       |  5 ++
 .../bfabric/experimental/upload_dataset.py    |  2 -
 bfabric_scripts/doc/changelog.md              |  4 +
 .../src/bfabric_scripts/cli/__main__.py       |  2 +
 .../src/bfabric_scripts/cli/cli_dataset.py    | 10 +++
 .../bfabric_scripts/cli/dataset/__init__.py   |  0
 .../bfabric_scripts/cli/dataset/download.py   | 49 +++++++++++
 .../src/bfabric_scripts/cli/dataset/show.py   | 54 +++++++++++++
 .../src/bfabric_scripts/cli/dataset/upload.py | 81 +++++++++++++++++++
 9 files changed, 205 insertions(+), 2 deletions(-)
 create mode 100644 bfabric_scripts/src/bfabric_scripts/cli/cli_dataset.py
 create mode 100644 bfabric_scripts/src/bfabric_scripts/cli/dataset/__init__.py
 create mode 100644 bfabric_scripts/src/bfabric_scripts/cli/dataset/download.py
 create mode 100644 bfabric_scripts/src/bfabric_scripts/cli/dataset/show.py
 create mode 100644 bfabric_scripts/src/bfabric_scripts/cli/dataset/upload.py

diff --git a/bfabric/src/bfabric/entities/dataset.py b/bfabric/src/bfabric/entities/dataset.py
index 4ec1ef10..0927801e 100644
--- a/bfabric/src/bfabric/entities/dataset.py
+++ b/bfabric/src/bfabric/entities/dataset.py
@@ -31,6 +31,11 @@ def to_polars(self) -> DataFrame:
             data.append(dict(zip(column_names, row_values)))
         return DataFrame(data)
 
+    @property
+    def types(self) -> dict[str, str]:
+        """Returns a dictionary mapping column names to their data types."""
+        return {x["name"]: x["type"] for x in self.data_dict["attribute"]}
+
     def write_csv(self, path: Path, separator: str = ",") -> None:
         """Writes the dataset to a csv file at `path`, using the specified column `separator`."""
         self.to_polars().write_csv(path, separator=separator)
diff --git a/bfabric/src/bfabric/experimental/upload_dataset.py b/bfabric/src/bfabric/experimental/upload_dataset.py
index f10c451f..20f75bbd 100644
--- a/bfabric/src/bfabric/experimental/upload_dataset.py
+++ b/bfabric/src/bfabric/experimental/upload_dataset.py
@@ -15,8 +15,6 @@ def polars_to_bfabric_type(dtype: pl.DataType) -> str | None:
         return "Integer"
     elif str(dtype).startswith("String"):
         return "String"
-    elif str(dtype).startswith("Float"):
-        return "Float"
     else:
         return "String"
 
diff --git a/bfabric_scripts/doc/changelog.md b/bfabric_scripts/doc/changelog.md
index 36190483..8095c6a3 100644
--- a/bfabric_scripts/doc/changelog.md
+++ b/bfabric_scripts/doc/changelog.md
@@ -10,6 +10,10 @@ Versioning currently follows `X.Y.Z` where
 
 ## \[Unreleased\]
 
+### Added
+
+- `bfabric-cli dataset {upload, download, show}` to replace the old dataset-related scripts.
+
 ## \[1.13.22\] - 2025-02-17
 
 ### Fixed
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/__main__.py b/bfabric_scripts/src/bfabric_scripts/cli/__main__.py
index c964dbbd..5621153d 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/__main__.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/__main__.py
@@ -1,6 +1,7 @@
 import cyclopts
 
 from bfabric_scripts.cli.cli_api import app as _app_api
+from bfabric_scripts.cli.cli_dataset import app as _app_dataset
 from bfabric_scripts.cli.cli_executable import app as _app_executable
 from bfabric_scripts.cli.cli_external_job import app as _app_external_job
 from bfabric_scripts.cli.cli_workunit import app as _app_workunit
@@ -10,6 +11,7 @@
 app.command(_app_workunit, name="workunit")
 app.command(_app_api, name="api")
 app.command(_app_executable, name="executable")
+app.command(_app_dataset, name="dataset")
 
 if __name__ == "__main__":
     app()
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_dataset.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_dataset.py
new file mode 100644
index 00000000..a7823953
--- /dev/null
+++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_dataset.py
@@ -0,0 +1,10 @@
+import cyclopts
+
+from bfabric_scripts.cli.dataset.download import cmd_dataset_download
+from bfabric_scripts.cli.dataset.upload import cmd_dataset_upload
+from bfabric_scripts.cli.dataset.show import cmd_dataset_show
+
+app = cyclopts.App(help="Read and update dataset entities in B-Fabric.")
+app.command(cmd_dataset_upload, name="upload")
+app.command(cmd_dataset_download, name="download")
+app.command(cmd_dataset_show, name="show")
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/dataset/__init__.py b/bfabric_scripts/src/bfabric_scripts/cli/dataset/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/dataset/download.py b/bfabric_scripts/src/bfabric_scripts/cli/dataset/download.py
new file mode 100644
index 00000000..ab84d2b2
--- /dev/null
+++ b/bfabric_scripts/src/bfabric_scripts/cli/dataset/download.py
@@ -0,0 +1,49 @@
+from enum import Enum
+from pathlib import Path
+
+import cyclopts
+import sys
+from loguru import logger
+from pydantic import BaseModel
+
+from bfabric import Bfabric
+from bfabric.entities import Dataset
+from bfabric.utils.cli_integration import use_client
+
+
+class OutputFormat(Enum):
+    CSV = "csv"
+    TSV = "tsv"
+    PARQUET = "parquet"
+
+
+@cyclopts.Parameter(name="*")
+class Params(BaseModel):
+    dataset_id: int
+    """The dataset ID to download."""
+    file: Path
+    """The file to download the dataset to."""
+    format: OutputFormat = OutputFormat.CSV
+    """The format to download the dataset in."""
+
+
+@use_client
+def cmd_dataset_download(params: Params, *, client: Bfabric) -> None:
+    """Download a dataset from B-Fabric."""
+    # Find the dataset
+    dataset = Dataset.find(id=params.dataset_id, client=client)
+    if not dataset:
+        msg = f"Dataset with id {params.dataset_id!r} not found."
+        raise ValueError(msg)
+
+    # Create the output directory if it does not exist
+    params.file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write the result
+    with logger.catch(onerror=lambda _: sys.exit(1)):
+        match params.format:
+            case OutputFormat.CSV | OutputFormat.TSV:
+                dataset.write_csv(params.file, separator="\t" if params.format == OutputFormat.TSV else ",")
+            case OutputFormat.PARQUET:
+                dataset.to_polars().write_parquet(params.file)
+    logger.info(f"Result written to {params.file} ({params.file.stat().st_size / 1024:.2f} KB)")
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/dataset/show.py b/bfabric_scripts/src/bfabric_scripts/cli/dataset/show.py
new file mode 100644
index 00000000..6c7b6b3c
--- /dev/null
+++ b/bfabric_scripts/src/bfabric_scripts/cli/dataset/show.py
@@ -0,0 +1,54 @@
+import bfabric.entities
+import inspect
+from enum import Enum
+
+import polars as pl
+import rich
+import yaml
+from rich.table import Table
+
+from bfabric import Bfabric
+from bfabric.entities import Dataset
+from bfabric.utils.cli_integration import use_client
+
+
+class OutputFormat(Enum):
+    TABLE = "table"
+    YAML = "YAML"
+
+
+def get_defined_entities():
+    return {name: klass for name, klass in inspect.getmembers(bfabric.entities, inspect.isclass)}
+
+
+def _print_table(dataframe: pl.DataFrame, types: dict[str, str], client: Bfabric) -> None:
+    table = Table(*dataframe.columns)
+    defined_entities = get_defined_entities()
+    for row in dataframe.rows():
+        out_row = []
+        for col, col_value in zip(dataframe.columns, row):
+            entity_class = defined_entities.get(types.get(col))
+            if entity_class is not None:
+                url = entity_class({"id": col_value}, client=client).web_url
+                out_row.append(f"[link={url}]{col_value}[/link]")
+            else:
+                out_row.append(col_value)
+        table.add_row(*out_row)
+    rich.print(table)
+
+
+def _print_yaml(dataframe: pl.DataFrame) -> None:
+    rich.print(yaml.safe_dump(dataframe.to_dicts()))
+
+
+@use_client
+def cmd_dataset_show(dataset_id: int, format: OutputFormat = OutputFormat.TABLE, *, client: Bfabric) -> None:
+    """Show a dataset in the console."""
+    dataset = Dataset.find(id=dataset_id, client=client)
+    types = dataset.types
+    dataframe = dataset.to_polars()
+
+    if format == OutputFormat.TABLE:
+        _print_table(dataframe, types, client)
+    elif format == OutputFormat.YAML:
+        _print_yaml(dataframe)
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/dataset/upload.py b/bfabric_scripts/src/bfabric_scripts/cli/dataset/upload.py
new file mode 100644
index 00000000..a7a6652b
--- /dev/null
+++ b/bfabric_scripts/src/bfabric_scripts/cli/dataset/upload.py
@@ -0,0 +1,81 @@
+from enum import Enum
+from pathlib import Path
+
+import cyclopts
+import polars as pl
+from pydantic import BaseModel
+from rich.pretty import pprint
+
+from bfabric import Bfabric
+from bfabric.experimental.upload_dataset import check_for_invalid_characters, polars_to_bfabric_dataset
+from bfabric.utils.cli_integration import use_client
+
+
+class InputFormat(Enum):
+    CSV = "csv"
+    PARQUET = "parquet"
+
+
+@cyclopts.Parameter(name="*")
+class Params(BaseModel):
+    file: Path
+    """The file to upload."""
+    container_id: int
+    """The container ID to upload the dataset to."""
+    dataset_name: str | None = None
+    """If set, the dataset will be given this name, otherwise it will default to the file name without the extension."""
+    workunit_id: int | None = None
+    """Register the dataset to have been created-by a workunit."""
+    forbidden_chars: str | None = ",\t"
+    """Characters which are not permitted in a cell (set to empty string to deactivate)."""
+
+
+@cyclopts.Parameter(name="*")
+class CsvParams(Params):
+    has_header: bool = True
+    """Whether the input file has a header."""
+    separator: str | None = None
+    """The separator to use in the CSV file."""
+
+
+cmd_dataset_upload = cyclopts.App(help="Upload a dataset to B-Fabric.")
+
+
+@cmd_dataset_upload.command
+@use_client
+def csv(params: CsvParams, *, client: Bfabric) -> None:
+    """Upload a CSV file as a B-Fabric dataset."""
+    params.separator = "," if params.separator is None else params.separator
+    table = pl.read_csv(params.file, separator=params.separator, has_header=params.has_header)
+    upload_table(table=table, params=params, client=client)
+
+
+@cmd_dataset_upload.command
+@use_client
+def tsv(params: CsvParams, *, client: Bfabric) -> None:
+    """Upload a TSV file as a B-Fabric dataset."""
+    params.separator = "\t" if params.separator is None else params.separator
+    # Defer to the CSV command
+    csv(params, client=client)
+
+
+@cmd_dataset_upload.command
+@use_client
+def parquet(params: Params, *, client: Bfabric) -> None:
+    """Upload a Parquet file as a B-Fabric dataset."""
+    table = pl.read_parquet(params.file)
+    upload_table(table=table, params=params, client=client)
+
+
+def upload_table(table: pl.DataFrame, params: Params, client: Bfabric) -> None:
+    if params.forbidden_chars:
+        check_for_invalid_characters(data=table, invalid_characters=params.forbidden_chars)
+
+    obj = polars_to_bfabric_dataset(table)
+    obj["name"] = params.dataset_name or params.file.stem
+    obj["containerid"] = params.container_id
+    if params.workunit_id is not None:
+        obj["workunitid"] = params.workunit_id
+    endpoint = "dataset"
+    res = client.save(endpoint=endpoint, obj=obj)
+    pprint(res[0])

From 02badb0e4d9dea94b427d83320cd2ea1b2f68d7e Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Mon, 17 Feb 2025 16:59:32 +0100
Subject: [PATCH 18/35] harmonize bfabric-cli structure

---
 ...list_not_available_proteomics_workunits.py |  4 +--
 .../src/bfabric_scripts/bfabric_logthis.py    |  2 +-
 .../src/bfabric_scripts/cli/__main__.py       | 18 ++++++-----
 .../{cli_api_log.py => _deprecated_log.py}    |  0
 .../{cli_api_save.py => _deprecated_save.py}  |  0
 .../cli/api/{cli_api_create.py => create.py}  |  4 +--
 .../cli/api/{cli_api_delete.py => delete.py}  |  5 +---
 .../cli/api/{cli_api_read.py => read.py}      | 10 +++----
 .../cli/api/{cli_api_update.py => update.py}  |  6 +---
 .../src/bfabric_scripts/cli/cli_api.py        | 30 +++++++++----------
 .../src/bfabric_scripts/cli/cli_dataset.py    |  8 ++---
 .../src/bfabric_scripts/cli/cli_executable.py |  6 ++--
 .../src/bfabric_scripts/cli/cli_workunit.py   | 10 +++----
 .../cli/workunit/export_definition.py         |  2 +-
 .../cli/workunit/not_available.py             |  2 +-
 tests/bfabric_cli/test_cli_api_read.py        |  2 +-
 16 files changed, 51 insertions(+), 58 deletions(-)
 rename bfabric_scripts/src/bfabric_scripts/cli/api/{cli_api_log.py => _deprecated_log.py} (100%)
 rename bfabric_scripts/src/bfabric_scripts/cli/api/{cli_api_save.py => _deprecated_save.py} (100%)
 rename bfabric_scripts/src/bfabric_scripts/cli/api/{cli_api_create.py => create.py} (89%)
 rename bfabric_scripts/src/bfabric_scripts/cli/api/{cli_api_delete.py => delete.py} (94%)
 rename bfabric_scripts/src/bfabric_scripts/cli/api/{cli_api_read.py => read.py} (97%)
 rename bfabric_scripts/src/bfabric_scripts/cli/api/{cli_api_update.py => update.py} (95%)

diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py b/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py
index e7ca1e09..f9000170 100755
--- a/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py
+++ b/bfabric_scripts/src/bfabric_scripts/bfabric_list_not_available_proteomics_workunits.py
@@ -16,7 +16,7 @@
 
 from bfabric.utils.cli_integration import setup_script_logging
 from bfabric_scripts.cli.workunit.not_available import (
-    list_not_available_proteomics_workunits,
+    cmd_workunit_not_available,
 )
 
 
@@ -26,7 +26,7 @@ def main() -> None:
     parser = ArgumentParser(description="Lists proteomics work units that are not available on bfabric.")
     parser.add_argument("--max-age", type=int, help="Max age of work units in days", default=14)
     args = parser.parse_args()
-    list_not_available_proteomics_workunits(max_age=args.max_age)
+    cmd_workunit_not_available(max_age=args.max_age)
 
 
 if __name__ == "__main__":
diff --git a/bfabric_scripts/src/bfabric_scripts/bfabric_logthis.py b/bfabric_scripts/src/bfabric_scripts/bfabric_logthis.py
index 245fb893..019acf70 100755
--- a/bfabric_scripts/src/bfabric_scripts/bfabric_logthis.py
+++ b/bfabric_scripts/src/bfabric_scripts/bfabric_logthis.py
@@ -9,7 +9,7 @@
 from typing import Literal
 
 from bfabric import Bfabric
-from bfabric_scripts.cli.api.cli_api_log import write_externaljob, write_workunit
+from bfabric_scripts.cli.api._deprecated_log import write_externaljob, write_workunit
 from bfabric.utils.cli_integration import use_client
 
 
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/__main__.py b/bfabric_scripts/src/bfabric_scripts/cli/__main__.py
index 5621153d..5f9d1c28 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/__main__.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/__main__.py
@@ -1,17 +1,19 @@
 import cyclopts
 
-from bfabric_scripts.cli.cli_api import app as _app_api
-from bfabric_scripts.cli.cli_dataset import app as _app_dataset
-from bfabric_scripts.cli.cli_executable import app as _app_executable
+from bfabric_scripts.cli.cli_api import cmd_api
+from bfabric_scripts.cli.cli_dataset import cmd_dataset
+from bfabric_scripts.cli.cli_executable import cmd_executable
 from bfabric_scripts.cli.cli_external_job import app as _app_external_job
-from bfabric_scripts.cli.cli_workunit import app as _app_workunit
+from bfabric_scripts.cli.cli_workunit import cmd_workunit
 
 app = cyclopts.App()
+app.command(cmd_api, name="api")
+app.command(cmd_dataset, name="dataset")
+app.command(cmd_executable, name="executable")
+app.command(cmd_workunit, name="workunit")
+
+# TODO delete after transitory release
 app.command(_app_external_job, name="external-job")
-app.command(_app_workunit, name="workunit")
-app.command(_app_api, name="api")
-app.command(_app_executable, name="executable")
-app.command(_app_dataset, name="dataset")
 
 if __name__ == "__main__":
     app()
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py b/bfabric_scripts/src/bfabric_scripts/cli/api/_deprecated_log.py
similarity index 100%
rename from bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_log.py
rename to bfabric_scripts/src/bfabric_scripts/cli/api/_deprecated_log.py
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_save.py b/bfabric_scripts/src/bfabric_scripts/cli/api/_deprecated_save.py
similarity index 100%
rename from bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_save.py
rename to bfabric_scripts/src/bfabric_scripts/cli/api/_deprecated_save.py
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py b/bfabric_scripts/src/bfabric_scripts/cli/api/create.py
similarity index 89%
rename from bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py
rename to bfabric_scripts/src/bfabric_scripts/cli/api/create.py
index d7446e3a..a369c0b7 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_create.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/create.py
@@ -1,7 +1,7 @@
 import cyclopts
 from cyclopts import Parameter
 from loguru import logger
-from pydantic import BaseModel, field_validator, Field, ValidationError
+from pydantic import BaseModel, field_validator, Field
 from rich.pretty import pprint
 
 from bfabric import Bfabric
@@ -29,7 +29,7 @@ def _must_not_contain_id(cls, value):
 @app.default
 @use_client
 @logger.catch(reraise=True)
-def create(params: Params, *, client: Bfabric) -> None:
+def cmd_api_create(params: Params, *, client: Bfabric) -> None:
     """Creates a new entity in B-Fabric."""
     attributes_dict = {attribute: value for attribute, value in params.attributes}
     result = client.save(params.endpoint, attributes_dict)
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_delete.py b/bfabric_scripts/src/bfabric_scripts/cli/api/delete.py
similarity index 94%
rename from bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_delete.py
rename to bfabric_scripts/src/bfabric_scripts/cli/api/delete.py
index 341687b6..7920f462 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_delete.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/delete.py
@@ -9,8 +9,6 @@
 from bfabric import Bfabric
 from bfabric.utils.cli_integration import use_client
 
-app = cyclopts.App()
-
 
 @cyclopts.Parameter(name="*")
 class Params(BaseModel):
@@ -30,10 +28,9 @@ def _perform_delete(client: Bfabric, endpoint: str, id: list[int]) -> None:
     logger.info(f"Entity with ID(s) {id} deleted successfully.")
 
 
-@app.default
 @logger.catch(reraise=True)
 @use_client
-def cli_api_delete(params: Params, *, client: Bfabric) -> None:
+def cmd_api_delete(params: Params, *, client: Bfabric) -> None:
     """Deletes entities from B-Fabric by id."""
     if params.no_confirm:
         _perform_delete(client=client, endpoint=params.endpoint, id=params.id)
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py b/bfabric_scripts/src/bfabric_scripts/cli/api/read.py
similarity index 97%
rename from bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py
rename to bfabric_scripts/src/bfabric_scripts/cli/api/read.py
index 55428c02..d6c18e59 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_read.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/read.py
@@ -8,16 +8,15 @@
 import polars as pl
 import pydantic
 import yaml
-from bfabric import Bfabric, BfabricClientConfig
-from bfabric.utils.polars_utils import flatten_relations
-from bfabric.utils.cli_integration import use_client
 from loguru import logger
 from pydantic import BaseModel
 from rich.console import Console
 from rich.syntax import Syntax
 from rich.table import Table
 
-app = cyclopts.App()
+from bfabric import Bfabric, BfabricClientConfig
+from bfabric.utils.cli_integration import use_client
+from bfabric.utils.polars_utils import flatten_relations
 
 
 class OutputFormat(Enum):
@@ -103,10 +102,9 @@ def render_output(results: list[dict[str, Any]], params: Params, client: Bfabric
         return result
 
 
-@app.default
 @use_client
 @logger.catch(reraise=True)
-def read(params: Annotated[Params, cyclopts.Parameter(name="*")], *, client: Bfabric) -> None | int:
+def cmd_api_read(params: Annotated[Params, cyclopts.Parameter(name="*")], *, client: Bfabric) -> None | int:
     """Reads entities from B-Fabric."""
     console_user = Console(stderr=True)
     console_user.print(params)
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_update.py b/bfabric_scripts/src/bfabric_scripts/cli/api/update.py
similarity index 95%
rename from bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_update.py
rename to bfabric_scripts/src/bfabric_scripts/cli/api/update.py
index 38b17682..e5202062 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/api/cli_api_update.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/api/update.py
@@ -1,4 +1,3 @@
-import cyclopts
 import rich
 import rich.prompt
 from cyclopts import Parameter
@@ -10,8 +9,6 @@
 from bfabric import Bfabric
 from bfabric.utils.cli_integration import use_client
 
-app = cyclopts.App()
-
 
 @Parameter(name="*")
 class Params(BaseModel):
@@ -25,10 +22,9 @@ class Params(BaseModel):
     """If set, the update will be performed without asking for confirmation."""
 
 
-@app.default
 @use_client
 @logger.catch(reraise=True)
-def update(params: Params, *, client: Bfabric) -> None:
+def cmd_api_update(params: Params, *, client: Bfabric) -> None:
     """Updates an existing entity in B-Fabric."""
     attributes_dict = _sanitize_attributes(params.attributes, params.entity_id)
     if not attributes_dict:
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
index b77ad3bb..dc72dcb4 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_api.py
@@ -1,19 +1,19 @@
 import cyclopts
 
-from bfabric_scripts.cli.api.cli_api_create import app as _cmd_create
-from bfabric_scripts.cli.api.cli_api_delete import app as _cmd_delete
-from bfabric_scripts.cli.api.cli_api_log import cmd as _cmd_log
-from bfabric_scripts.cli.api.cli_api_read import app as _cmd_read
-from bfabric_scripts.cli.api.cli_api_save import app as _cmd_save
-from bfabric_scripts.cli.api.cli_api_update import app as _cmd_update
+from bfabric_scripts.cli.api._deprecated_log import cmd as _cmd_log
+from bfabric_scripts.cli.api._deprecated_save import app as _cmd_save
+from bfabric_scripts.cli.api.create import cmd_api_create
+from bfabric_scripts.cli.api.delete import cmd_api_delete
+from bfabric_scripts.cli.api.read import cmd_api_read
+from bfabric_scripts.cli.api.update import cmd_api_update
 
-app = cyclopts.App()
-app.command(_cmd_create, name="create")
-app.command(_cmd_delete, name="delete")
-app.command(_cmd_read, name="read")
-app.command(_cmd_update, name="update")
+cmd_api = cyclopts.App(help="Commands for interacting with B-Fabric API directly.")
+cmd_api.command(cmd_api_create, name="create")
+cmd_api.command(cmd_api_delete, name="delete")
+cmd_api.command(cmd_api_read, name="read")
+cmd_api.command(cmd_api_update, name="update")
 
-# TODO delete or move
-app.command(_cmd_log, name="log")
-# TODO delete
-app.command(_cmd_save, name="save")
+# TODO delete after transitory release
+cmd_api.command(_cmd_log, name="log")
+# TODO delete after transitory release
+cmd_api.command(_cmd_save, name="save")
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_dataset.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_dataset.py
index a7823953..b7c3bf13 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/cli_dataset.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_dataset.py
@@ -4,7 +4,7 @@
 from bfabric_scripts.cli.dataset.upload import cmd_dataset_upload
 from bfabric_scripts.cli.dataset.show import cmd_dataset_show
 
-app = cyclopts.App(help="Read and update dataset entities in B-Fabric.")
-app.command(cmd_dataset_upload, name="upload")
-app.command(cmd_dataset_download, name="download")
-app.command(cmd_dataset_show, name="show")
+cmd_dataset = cyclopts.App(help="Read and update dataset entities in B-Fabric.")
+cmd_dataset.command(cmd_dataset_upload, name="upload")
+cmd_dataset.command(cmd_dataset_download, name="download")
+cmd_dataset.command(cmd_dataset_show, name="show")
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_executable.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_executable.py
index ff24ea21..750fa2b0 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/cli_executable.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_executable.py
@@ -3,6 +3,6 @@
 from bfabric_scripts.cli.executable.show import cmd_executable_show
 from bfabric_scripts.cli.executable.upload import cmd_executable_upload
 
-app = cyclopts.App()
-app.command(cmd_executable_show, name="show")
-app.command(cmd_executable_upload, name="upload")
+cmd_executable = cyclopts.App(help="Read and write executable entities in B-Fabric.")
+cmd_executable.command(cmd_executable_show, name="show")
+cmd_executable.command(cmd_executable_upload, name="upload")
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/cli_workunit.py b/bfabric_scripts/src/bfabric_scripts/cli/cli_workunit.py
index a24e2ee1..7f8f2aa9 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/cli_workunit.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/cli_workunit.py
@@ -1,10 +1,10 @@
 import cyclopts
 
-from bfabric_scripts.cli.workunit.export_definition import export_definition
+from bfabric_scripts.cli.workunit.export_definition import cmd_workunit_export_definition
 from bfabric_scripts.cli.workunit.not_available import (
-    list_not_available_proteomics_workunits,
+    cmd_workunit_not_available,
 )
 
-app = cyclopts.App()
-app.command(list_not_available_proteomics_workunits, name="not-available")
-app.command(export_definition, name="export-definition")
+cmd_workunit = cyclopts.App(help="Read workunit entities in B-Fabric.")
+cmd_workunit.command(cmd_workunit_not_available, name="not-available")
+cmd_workunit.command(cmd_workunit_export_definition, name="export-definition")
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/workunit/export_definition.py b/bfabric_scripts/src/bfabric_scripts/cli/workunit/export_definition.py
index 02c35ff2..ef56121f 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/workunit/export_definition.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/workunit/export_definition.py
@@ -8,7 +8,7 @@
 
 
 @use_client
-def export_definition(workunit_id: int, target_path: Path | None = None, *, client: Bfabric) -> None:
+def cmd_workunit_export_definition(workunit_id: int, target_path: Path | None = None, *, client: Bfabric) -> None:
     """Exports a workunit_definition.yml file for the specified workunit.
 
     :param workunit_id: the workunit ID
diff --git a/bfabric_scripts/src/bfabric_scripts/cli/workunit/not_available.py b/bfabric_scripts/src/bfabric_scripts/cli/workunit/not_available.py
index ff88439e..90c22d08 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/workunit/not_available.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/workunit/not_available.py
@@ -74,7 +74,7 @@ def filter_workunits_by_user(workunits: list[Workunit], exclude_user: list[str]
 
 
 @use_client
-def list_not_available_proteomics_workunits(
+def cmd_workunit_not_available(
     *,
     client: Bfabric,
     max_age: float = 14.0,
diff --git a/tests/bfabric_cli/test_cli_api_read.py b/tests/bfabric_cli/test_cli_api_read.py
index 2b43d543..30ebd6e1 100644
--- a/tests/bfabric_cli/test_cli_api_read.py
+++ b/tests/bfabric_cli/test_cli_api_read.py
@@ -5,7 +5,7 @@
 import yaml
 from bfabric import Bfabric
 from bfabric.results.result_container import ResultContainer
-from bfabric_scripts.cli.api.cli_api_read import (
+from bfabric_scripts.cli.api.read import (
     Params,
     OutputFormat,
     perform_query,

From e5d69a81b9839fbd3f2bc84680a1af2559dcbec5 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Mon, 17 Feb 2025 17:14:12 +0100
Subject: [PATCH 19/35] harmonize bfabric-app-runner structure

---
 .../src/bfabric_app_runner/cli/__main__.py    | 50 +++++++++++++++----
 .../src/bfabric_app_runner/cli/app.py         | 20 +++-----
 .../src/bfabric_app_runner/cli/chunk.py       | 35 +++++--------
 .../src/bfabric_app_runner/cli/inputs.py      | 40 +++++++--------
 .../src/bfabric_app_runner/cli/outputs.py     | 24 ++++-----
 .../src/bfabric_app_runner/cli/validate.py    | 15 ++----
 6 files changed, 92 insertions(+), 92 deletions(-)

diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py
index 0545aa83..2998e688 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py
@@ -4,11 +4,16 @@
 
 import cyclopts
 
-from bfabric_app_runner.cli.app import app_app
-from bfabric_app_runner.cli.chunk import app_chunk
-from bfabric_app_runner.cli.inputs import app_inputs
-from bfabric_app_runner.cli.outputs import app_outputs
-from bfabric_app_runner.cli.validate import app_validate
+from bfabric_app_runner.cli.app import cmd_app_run, cmd_app_dispatch
+from bfabric_app_runner.cli.chunk import cmd_chunk_run_all, cmd_chunk_outputs, cmd_chunk_process
+from bfabric_app_runner.cli.inputs import cmd_inputs_prepare, cmd_inputs_clean, cmd_inputs_list
+from bfabric_app_runner.cli.outputs import cmd_outputs_register, cmd_outputs_register_single_file
+from bfabric_app_runner.cli.validate import (
+    cmd_validate_inputs_spec,
+    cmd_validate_outputs_spec,
+    cmd_validate_app_spec,
+    cmd_validate_app_spec_template,
+)
 
 package_version = importlib.metadata.version("bfabric_app_runner")
 
@@ -16,11 +21,36 @@
     help="Provides an entrypoint to app execution.\n\nFunctionality/API under active development!",
     version=package_version,
 )
-app.command(app_inputs)
-app.command(app_outputs)
-app.command(app_app)
-app.command(app_chunk)
-app.command(app_validate)
+
+cmd_app = cyclopts.App("app", help="Run an app.")
+cmd_app.command(cmd_app_dispatch, name="dispatch")
+cmd_app.command(cmd_app_run, name="run")
+app.command(cmd_app)
+
+cmd_inputs = cyclopts.App("inputs", help="Prepare input files for an app.")
+cmd_inputs.command(cmd_inputs_clean, name="clean")
+cmd_inputs.command(cmd_inputs_list, name="list")
+cmd_inputs.command(cmd_inputs_prepare, name="prepare")
+app.command(cmd_inputs)
+
+cmd_outputs = cyclopts.App("outputs", help="Register output files of an app.")
+cmd_outputs.command(cmd_outputs_register, name="register")
+cmd_outputs.command(cmd_outputs_register_single_file, name="register-single-file")
+app.command(cmd_outputs)
+
+cmd_chunk = cyclopts.App("chunk", help="Run an app on a chunk. You can create the chunks with `app dispatch`.")
+cmd_chunk.command(cmd_chunk_outputs, name="outputs")
+cmd_chunk.command(cmd_chunk_process, name="process")
+cmd_chunk.command(cmd_chunk_run_all, name="run-all")
+app.command(cmd_chunk)
+
+cmd_validate = cyclopts.App("validate", help="Validate yaml files.")
+cmd_validate.command(cmd_validate_app_spec, name="app-spec")
+cmd_validate.command(cmd_validate_app_spec_template, name="app-spec-template")
+cmd_validate.command(cmd_validate_inputs_spec, name="inputs-spec")
+cmd_validate.command(cmd_validate_outputs_spec, name="outputs-spec")
+
+app.command(cmd_validate)
 
 if __name__ == "__main__":
     app()
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
index e2f741a6..222c3617 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
@@ -4,31 +4,27 @@
 import importlib.resources
 from pathlib import Path
 
-import cyclopts
 from loguru import logger
 
 from bfabric import Bfabric
 from bfabric.experimental.entity_lookup_cache import EntityLookupCache
-from bfabric.utils.cli_integration import setup_script_logging
+from bfabric.utils.cli_integration import use_client
 from bfabric_app_runner.app_runner.resolve_app import load_workunit_information
 from bfabric_app_runner.app_runner.runner import run_app, Runner
 
-app_app = cyclopts.App("app", help="Run an app.")
 
-
-@app_app.command()
-def run(
+@use_client
+def cmd_app_run(
     app_spec: Path,
     work_dir: Path,
     workunit_ref: int | Path,
     *,
     ssh_user: str | None = None,
     read_only: bool = False,
+    client: Bfabric,
 ) -> None:
     """Runs all stages of an app."""
     # TODO doc
-    setup_script_logging()
-    client = Bfabric.from_config()
     app_version, workunit_ref = load_workunit_information(app_spec, client, work_dir, workunit_ref)
 
     copy_dev_makefile(work_dir=work_dir)
@@ -46,11 +42,13 @@ def run(
     )
 
 
-@app_app.command()
-def dispatch(
+@use_client
+def cmd_app_dispatch(
     app_spec: Path,
     work_dir: Path,
     workunit_ref: int | Path,
+    *,
+    client: Bfabric,
 ) -> None:
     """Create chunks, which can be processed individually.
 
@@ -58,10 +56,8 @@ def dispatch(
     :param work_dir: Path to the work directory.
     :param workunit_ref: Reference to the workunit (ID or YAML file path).
     """
-    setup_script_logging()
     work_dir = work_dir.resolve()
     # TODO set workunit to processing? (i.e. add read-only option here)
-    client = Bfabric.from_config()
 
     app_version, workunit_ref = load_workunit_information(app_spec, client, work_dir, workunit_ref)
 
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py b/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py
index aec330b1..e21d55fd 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py
@@ -2,27 +2,24 @@
 
 from pathlib import Path
 
-import cyclopts
-
-from bfabric_app_runner.app_runner.resolve_app import load_workunit_information
-from bfabric_app_runner.app_runner.runner import run_app, Runner
-from bfabric_app_runner.output_registration import register_outputs
 from bfabric import Bfabric
-from bfabric.utils.cli_integration import setup_script_logging
 from bfabric.experimental.entity_lookup_cache import EntityLookupCache
 from bfabric.experimental.workunit_definition import WorkunitDefinition
-
-app_chunk = cyclopts.App("chunk", help="Run an app on a chunk. You can create the chunks with `app dispatch`.")
+from bfabric.utils.cli_integration import use_client
+from bfabric_app_runner.app_runner.resolve_app import load_workunit_information
+from bfabric_app_runner.app_runner.runner import run_app, Runner
+from bfabric_app_runner.output_registration import register_outputs
 
 
-@app_chunk.command()
-def run_all(
+@use_client
+def cmd_chunk_run_all(
     app_spec: Path,
     work_dir: Path,
     workunit_ref: int | Path,
     *,
     ssh_user: str | None = None,
     read_only: bool = False,
+    client: Bfabric,
 ) -> None:
     """Run all chunks, including input preparation, processing, and output registration.
 
@@ -32,9 +29,6 @@ def run_all(
     :param ssh_user: SSH user to use for downloading the input files, instead of the current user.
     :param read_only: If True, results will not be registered and the workunit status will not be changed.
     """
-    setup_script_logging()
-    client = Bfabric.from_config()
-
     app_version, workunit_ref = load_workunit_information(
         app_spec=app_spec, client=client, work_dir=work_dir, workunit_ref=workunit_ref
     )
@@ -50,16 +44,14 @@ def run_all(
     )
 
 
-@app_chunk.command()
-def process(app_spec: Path, chunk_dir: Path) -> None:
+@use_client
+def cmd_chunk_process(app_spec: Path, chunk_dir: Path, *, client: Bfabric) -> None:
     """Process a chunk.
 
     Note that the input files must be prepared before running this command.
     :param app_spec: Path to the app spec file.
     :param chunk_dir: Path to the chunk directory.
     """
-    setup_script_logging()
-    client = Bfabric.from_config()
     chunk_dir = chunk_dir.resolve()
 
     # TODO this lookup of workunit_definition is very problematic now! FIX NEEDED
@@ -72,8 +64,8 @@ def process(app_spec: Path, chunk_dir: Path) -> None:
         runner.run_process(chunk_dir=chunk_dir)
 
 
-@app_chunk.command()
-def outputs(
+@use_client
+def cmd_chunk_outputs(
     app_spec: Path,
     chunk_dir: Path,
     workunit_ref: int | Path,
@@ -82,6 +74,7 @@ def outputs(
     force_storage: Path | None = None,
     read_only: bool = False,
     reuse_default_resource: bool = True,
+    client: Bfabric,
 ) -> None:
     """Register the output files of a chunk.
 
@@ -92,9 +85,7 @@ def outputs(
     :param read_only: If True, the workunit will not be set to processing.
     :param reuse_default_resource: If True, the default resource will be reused for the output files. (recommended)
     """
-    # TODO redundant with "outputs register"
-    setup_script_logging()
-    client = Bfabric.from_config()
+    # TODO redundant functionality with "outputs register"
     chunk_dir = chunk_dir.resolve()
 
     app_version, workunit_ref = load_workunit_information(
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py b/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py
index 6c230ab6..303cc8e5 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py
@@ -2,8 +2,8 @@
 
 from pathlib import Path
 
-import cyclopts
-
+from bfabric import Bfabric
+from bfabric.utils.cli_integration import use_client
 from bfabric_app_runner.input_preparation import prepare_folder
 from bfabric_app_runner.input_preparation.integrity import IntegrityState
 from bfabric_app_runner.input_preparation.list_inputs import (
@@ -12,19 +12,16 @@
     FileState,
 )
 from bfabric_app_runner.specs.inputs_spec import InputsSpec
-from bfabric import Bfabric
-from bfabric.utils.cli_integration import setup_script_logging
 
-app_inputs = cyclopts.App("inputs", help="Prepare input files for an app.")
 
-
-@app_inputs.command()
-def prepare(
+@use_client
+def cmd_inputs_prepare(
     inputs_yaml: Path,
     target_folder: Path | None = None,
     *,
     ssh_user: str | None = None,
     filter: str | None = None,
+    client: Bfabric,
 ) -> None:
     """Prepare the input files by downloading and generating them (if necessary).
 
@@ -33,8 +30,6 @@ def prepare(
     :param ssh_user: SSH user to use for downloading the input files, instead of the current user.
     :param filter: only this input file will be prepared.
     """
-    setup_script_logging()
-    client = Bfabric.from_config()
     prepare_folder(
         inputs_yaml=inputs_yaml,
         target_folder=target_folder,
@@ -45,12 +40,13 @@ def prepare(
     )
 
 
-@app_inputs.command()
-def clean(
+@use_client
+def cmd_inputs_clean(
     inputs_yaml: Path,
     target_folder: Path | None = None,
     *,
     filter: str | None = None,
+    client: Bfabric,
 ) -> None:
     """Removes all local copies of input files.
 
@@ -58,8 +54,6 @@ def clean(
     :param target_folder: Path to the target folder where the input files should be removed.
     :param filter: only this input file will be removed.
     """
-    setup_script_logging()
-    client = Bfabric.from_config()
     # TODO clean shouldn't even need all these arguments, this could be refactored later
     prepare_folder(
         inputs_yaml=inputs_yaml,
@@ -75,9 +69,9 @@ def get_inputs_and_print(
     inputs_yaml: Path,
     target_folder: Path | None,
     check: bool,
+    client: Bfabric,
 ) -> list[FileState]:
     """Reads the input files, performing integrity checks if requested, and prints the results."""
-    client = Bfabric.from_config()
     input_states = list_input_states(
         specs=InputsSpec.read_yaml_old(inputs_yaml),
         target_folder=target_folder or Path(),
@@ -88,11 +82,13 @@ def get_inputs_and_print(
     return input_states
 
 
-@app_inputs.command(name="list")
-def list_(
+@use_client
+def cmd_inputs_list(
     inputs_yaml: Path,
     target_folder: Path | None = None,
     check: bool = False,
+    *,
+    client: Bfabric,
 ) -> None:
     """Lists the input files for an app.
 
@@ -100,14 +96,15 @@ def list_(
     :param target_folder: Path to the target folder where the input files should be located, if different from the
         file containing the inputs.yml file.
     """
-    setup_script_logging()
-    get_inputs_and_print(inputs_yaml=inputs_yaml, target_folder=target_folder, check=check)
+    get_inputs_and_print(inputs_yaml=inputs_yaml, target_folder=target_folder, check=check, client=client)
 
 
-@app_inputs.command()
+@use_client
 def check(
     inputs_yaml: Path,
     target_folder: Path | None = None,
+    *,
+    client: Bfabric,
 ) -> None:
     """Checks if the input files are present and have the correct content.
 
@@ -117,8 +114,7 @@ def check(
     :param target_folder: Path to the target folder where the input files should be located, if different from the
         file containing the inputs.yml file.
     """
-    setup_script_logging()
-    input_states = get_inputs_and_print(inputs_yaml=inputs_yaml, target_folder=target_folder, check=True)
+    input_states = get_inputs_and_print(inputs_yaml=inputs_yaml, target_folder=target_folder, check=True, client=client)
     invalid_states = {state.integrity for state in input_states if state.integrity != IntegrityState.Correct}
     if invalid_states:
         print(f"Encountered invalid input states: {invalid_states}")
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py b/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py
index 6e53471c..fe665d8c 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py
@@ -2,16 +2,13 @@
 
 from pathlib import Path
 
-import cyclopts
 from rich.pretty import pprint
 
-from bfabric_app_runner.output_registration.register import register_all
-from bfabric_app_runner.specs.outputs_spec import OutputsSpec, CopyResourceSpec, UpdateExisting
 from bfabric import Bfabric
-from bfabric.utils.cli_integration import setup_script_logging
 from bfabric.experimental.workunit_definition import WorkunitDefinition
-
-app_outputs = cyclopts.App("outputs", help="Register output files for an app.")
+from bfabric.utils.cli_integration import use_client
+from bfabric_app_runner.output_registration.register import register_all
+from bfabric_app_runner.specs.outputs_spec import OutputsSpec, CopyResourceSpec, UpdateExisting
 
 
 def _get_workunit_definition(client: Bfabric, workunit_ref: int | Path) -> WorkunitDefinition:
@@ -21,19 +18,18 @@ def _get_workunit_definition(client: Bfabric, workunit_ref: int | Path) -> Worku
     return WorkunitDefinition.from_ref(workunit=workunit_ref, client=client, cache_file=None)
 
 
-@app_outputs.command()
-def register(
+@use_client
+def cmd_outputs_register(
     outputs_yaml: Path,
     workunit_ref: int | Path,
     *,
     ssh_user: str | None = None,
     force_storage: Path | None = None,
+    client: Bfabric,
     # TODO
     reuse_default_resource: bool = True,
 ) -> None:
     """Register the output files of a workunit."""
-    setup_script_logging()
-    client = Bfabric.from_config()
     specs_list = OutputsSpec.read_yaml(outputs_yaml)
     register_all(
         client=client,
@@ -45,8 +41,8 @@ def register(
     )
 
 
-@app_outputs.command()
-def register_single_file(
+@use_client
+def cmd_outputs_register_single_file(
     local_path: Path,
     *,
     workunit_ref: int | Path,
@@ -56,14 +52,12 @@ def register_single_file(
     ssh_user: str | None = None,
     force_storage: Path | None = None,
     reuse_default_resource: bool = False,
+    client: Bfabric,
 ) -> None:
     """Register a single file in the workunit.
 
     In general, it is recommended to use the `register` command instead of this one and declare files using YAML.
     """
-    setup_script_logging()
-    client = Bfabric.from_config()
-
     if store_entry_path is None:
         store_entry_path = local_path.name
 
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py b/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py
index 76406a5d..3fba6153 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py
@@ -2,7 +2,6 @@
 
 from pathlib import Path
 
-import cyclopts
 import yaml
 from rich.pretty import pprint
 
@@ -10,32 +9,26 @@
 from bfabric_app_runner.specs.inputs_spec import InputsSpec
 from bfabric_app_runner.specs.outputs_spec import OutputsSpec
 
-app_validate = cyclopts.App("validate", help="Validate yaml files.")
 
-
-@app_validate.command()
-def app_spec_template(yaml_file: Path) -> None:
+def cmd_validate_app_spec_template(yaml_file: Path) -> None:
     """Validate an app spec file."""
     app_spec_file = AppSpecTemplate.model_validate(yaml.safe_load(yaml_file.read_text()))
     pprint(app_spec_file)
 
 
-@app_validate.command()
-def app_spec(app_yaml: Path, app_id: str = "x", app_name: str = "y") -> None:
+def cmd_validate_app_spec(app_yaml: Path, app_id: str = "x", app_name: str = "y") -> None:
     """Validates the app versions by expanding the relevant config info."""
     versions = AppSpec.load_yaml(app_yaml, app_id=app_id, app_name=app_name)
     pprint(versions)
 
 
-@app_validate.command()
-def inputs_spec(yaml_file: Path) -> None:
+def cmd_validate_inputs_spec(yaml_file: Path) -> None:
     """Validate an inputs spec file."""
     inputs_spec = InputsSpec.model_validate(yaml.safe_load(yaml_file.read_text()))
     pprint(inputs_spec)
 
 
-@app_validate.command()
-def outputs_spec(yaml_file: Path) -> None:
+def cmd_validate_outputs_spec(yaml_file: Path) -> None:
     """Validate an outputs spec file."""
     outputs_spec = OutputsSpec.model_validate(yaml.safe_load(yaml_file.read_text()))
     pprint(outputs_spec)

From 50b1a9bc18a53fdf918dd4d931a41228b569e57d Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Mon, 17 Feb 2025 17:17:35 +0100
Subject: [PATCH 20/35] refactoring fix

---
 bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py | 3 ++-
 bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py
index 2998e688..94bfe48e 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py
@@ -6,7 +6,7 @@
 
 from bfabric_app_runner.cli.app import cmd_app_run, cmd_app_dispatch
 from bfabric_app_runner.cli.chunk import cmd_chunk_run_all, cmd_chunk_outputs, cmd_chunk_process
-from bfabric_app_runner.cli.inputs import cmd_inputs_prepare, cmd_inputs_clean, cmd_inputs_list
+from bfabric_app_runner.cli.inputs import cmd_inputs_prepare, cmd_inputs_clean, cmd_inputs_list, cmd_inputs_check
 from bfabric_app_runner.cli.outputs import cmd_outputs_register, cmd_outputs_register_single_file
 from bfabric_app_runner.cli.validate import (
     cmd_validate_inputs_spec,
@@ -28,6 +28,7 @@
 app.command(cmd_app)
 
 cmd_inputs = cyclopts.App("inputs", help="Prepare input files for an app.")
+cmd_inputs.command(cmd_inputs_check, name="check")
 cmd_inputs.command(cmd_inputs_clean, name="clean")
 cmd_inputs.command(cmd_inputs_list, name="list")
 cmd_inputs.command(cmd_inputs_prepare, name="prepare")
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py b/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py
index 303cc8e5..282fc267 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py
@@ -100,7 +100,7 @@ def cmd_inputs_list(
 
 
 @use_client
-def check(
+def cmd_inputs_check(
     inputs_yaml: Path,
     target_folder: Path | None = None,
     *,

From 3db5bf7d27c684ab1fe37e512ec34dd18ad1d65c Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Mon, 17 Feb 2025 17:18:40 +0100
Subject: [PATCH 21/35] avoid deprecated functionality

---
 bfabric/src/bfabric/rest/token_data.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/bfabric/src/bfabric/rest/token_data.py b/bfabric/src/bfabric/rest/token_data.py
index 13172ea7..cccc65be 100644
--- a/bfabric/src/bfabric/rest/token_data.py
+++ b/bfabric/src/bfabric/rest/token_data.py
@@ -1,9 +1,10 @@
 from __future__ import annotations
+
 from datetime import datetime
 from typing import TYPE_CHECKING
 
 import requests
-from pydantic import BaseModel, Field, SecretStr
+from pydantic import BaseModel, Field, SecretStr, ConfigDict
 
 if TYPE_CHECKING:
     from bfabric import BfabricClientConfig
@@ -12,6 +13,10 @@
 class TokenData(BaseModel):
     """Parsed token data from the B-Fabric token validation endpoint."""
 
+    model_config = ConfigDict(
+        populate_by_name=True, str_strip_whitespace=True, json_encoders={datetime: lambda v: v.isoformat()}
+    )
+
     job_id: int = Field(alias="jobId")
     application_id: int = Field(alias="applicationId")
 
@@ -24,11 +29,6 @@ class TokenData(BaseModel):
     token_expires: datetime = Field(alias="expiryDateTime")
     environment: str
 
-    class Config:
-        populate_by_name = True
-        str_strip_whitespace = True
-        json_encoders = {datetime: lambda v: v.isoformat()}
-
 
 def get_token_data(client_config: BfabricClientConfig, token: str) -> TokenData:
     """Returns the token data for the provided token.

From d1d6086cdf962175d007d0666a64e3b861e79be0 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Mon, 17 Feb 2025 17:20:50 +0100
Subject: [PATCH 22/35] add package version information

---
 bfabric_scripts/src/bfabric_scripts/cli/__main__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bfabric_scripts/src/bfabric_scripts/cli/__main__.py b/bfabric_scripts/src/bfabric_scripts/cli/__main__.py
index 5f9d1c28..af3893e9 100644
--- a/bfabric_scripts/src/bfabric_scripts/cli/__main__.py
+++ b/bfabric_scripts/src/bfabric_scripts/cli/__main__.py
@@ -1,3 +1,5 @@
+import importlib.metadata
+
 import cyclopts
 
 from bfabric_scripts.cli.cli_api import cmd_api
@@ -6,7 +8,9 @@
 from bfabric_scripts.cli.cli_external_job import app as _app_external_job
 from bfabric_scripts.cli.cli_workunit import cmd_workunit
 
-app = cyclopts.App()
+package_version = importlib.metadata.version("bfabric_scripts")
+
+app = cyclopts.App(version=package_version)
 app.command(cmd_api, name="api")
 app.command(cmd_dataset, name="dataset")
 app.command(cmd_executable, name="executable")

From 5b4a0e9eea1f0e4426fc5e34f7e1a8bcd977fc97 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Mon, 17 Feb 2025 17:22:08 +0100
Subject: [PATCH 23/35] update test

---
 tests/bfabric_cli/test_cli_api_read.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/bfabric_cli/test_cli_api_read.py b/tests/bfabric_cli/test_cli_api_read.py
index 30ebd6e1..2c5b76dc 100644
--- a/tests/bfabric_cli/test_cli_api_read.py
+++ b/tests/bfabric_cli/test_cli_api_read.py
@@ -9,7 +9,7 @@
     Params,
     OutputFormat,
     perform_query,
-    read,
+    cmd_api_read,
     render_output,
     _determine_output_columns,
 )
@@ -95,7 +95,7 @@ def test_render_yaml_output(self, mocker, sample_results):
     def test_render_tsv_output(self, sample_results, mocker):
         # Arrange
         params = Params(endpoint="resource", columns=["id", "name"], format=OutputFormat.TSV)
-        mock_flatten = mocker.patch("bfabric_scripts.cli.api.cli_api_read.flatten_relations")
+        mock_flatten = mocker.patch("bfabric_scripts.cli.api.read.flatten_relations")
         mock_df = mocker.Mock()
         mock_flatten.return_value = mock_df
 
@@ -148,7 +148,7 @@ def test_determine_output_columns_invalid_max_columns(self):
 class TestReadFunction:
     def test_read_json_output(self, mock_client, mock_console, sample_results, mocker):
         # Arrange
-        mock_perform_query = mocker.patch("bfabric_scripts.cli.api.cli_api_read.perform_query")
+        mock_perform_query = mocker.patch("bfabric_scripts.cli.api.read.perform_query")
         mock_perform_query.return_value = sample_results
 
         params = Params(
@@ -156,7 +156,7 @@ def test_read_json_output(self, mock_client, mock_console, sample_results, mocke
         )
 
         # Act
-        result = read(params, client=mock_client)
+        result = cmd_api_read(params, client=mock_client)
 
         # Assert
         mock_perform_query.assert_called_once_with(params=params, client=mock_client, console_user=mocker.ANY)
@@ -165,7 +165,7 @@ def test_read_json_output(self, mock_client, mock_console, sample_results, mocke
     def test_read_with_file_output(self, mock_client, mock_console, sample_results, mocker, tmp_path):
         # Arrange
         output_file = tmp_path / "test_output.json"
-        mock_perform_query = mocker.patch("bfabric_scripts.cli.api.cli_api_read.perform_query")
+        mock_perform_query = mocker.patch("bfabric_scripts.cli.api.read.perform_query")
         mock_perform_query.return_value = sample_results
 
         params = Params(
@@ -177,7 +177,7 @@ def test_read_with_file_output(self, mock_client, mock_console, sample_results,
         )
 
         # Act
-        result = read(params, client=mock_client)
+        result = cmd_api_read(params, client=mock_client)
 
         # Assert
         assert result is None
@@ -189,9 +189,9 @@ def test_read_with_file_output(self, mock_client, mock_console, sample_results,
 
     def test_read_with_tsv_format(self, mock_client, mock_console, sample_results, mocker):
         # Arrange
-        mock_perform_query = mocker.patch("bfabric_scripts.cli.api.cli_api_read.perform_query")
+        mock_perform_query = mocker.patch("bfabric_scripts.cli.api.read.perform_query")
         mock_perform_query.return_value = sample_results
-        mock_flatten = mocker.patch("bfabric_scripts.cli.api.cli_api_read.flatten_relations")
+        mock_flatten = mocker.patch("bfabric_scripts.cli.api.read.flatten_relations")
         mock_df = mocker.Mock()
         mock_df.write_csv.return_value = "mocked,csv,content"
         mock_flatten.return_value = mock_df
@@ -201,7 +201,7 @@ def test_read_with_tsv_format(self, mock_client, mock_console, sample_results, m
         )
 
         # Act
-        result = read(params, client=mock_client)
+        result = cmd_api_read(params, client=mock_client)
 
         # Assert
         mock_perform_query.assert_called_once()
@@ -211,7 +211,7 @@ def test_read_with_tsv_format(self, mock_client, mock_console, sample_results, m
 
     def test_read_with_invalid_file_output(self, mock_client, mock_console, sample_results, mocker):
         # Arrange
-        mock_perform_query = mocker.patch("bfabric_scripts.cli.api.cli_api_read.perform_query")
+        mock_perform_query = mocker.patch("bfabric_scripts.cli.api.read.perform_query")
         mock_perform_query.return_value = sample_results
 
         # Using TABLE_RICH format which doesn't support file output
@@ -224,7 +224,7 @@ def test_read_with_invalid_file_output(self, mock_client, mock_console, sample_r
         )
 
         # Act
-        result = read(params, client=mock_client)
+        result = cmd_api_read(params, client=mock_client)
 
         # Assert
         assert result == 1  # Should return error code 1

From 5a7688a2bef3d64790bfa4e22f1ebba6bbd4d6d1 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Tue, 18 Feb 2025 09:11:24 +0100
Subject: [PATCH 24/35] do not build app runner automatically anymore

---
 .github/workflows/build_app_runner.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/build_app_runner.yml b/.github/workflows/build_app_runner.yml
index feea5ebe..62a51d20 100644
--- a/.github/workflows/build_app_runner.yml
+++ b/.github/workflows/build_app_runner.yml
@@ -1,10 +1,6 @@
 name: Build App Runner
 on:
   workflow_dispatch:
-  release:
-    types: [published]
-  pull_request:
-    branches: [stable]
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
@@ -32,7 +28,6 @@ jobs:
     # - It's a release with valid tag
     if: >
       ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && needs.check_release_tag.outputs.is_valid_tag == 'true') }}
-
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

From 175fde637ee7bdca423b5a80c2da38db545388d0 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Tue, 18 Feb 2025 09:20:16 +0100
Subject: [PATCH 25/35] move the docs to a consistent location

---
 {docs => bfabric/docs}/changelog.md        |  0
 {docs => bfabric/docs}/contribute.md       |  0
 {docs => bfabric/docs}/entities.md         |  0
 {docs => bfabric/docs}/good_to_know.md     |  0
 {docs => bfabric/docs}/index.md            |  0
 {docs => bfabric/docs}/old/cheatsheet.md   |  0
 {docs => bfabric/docs}/old/faq.md          |  0
 mkdocs.yml => bfabric/mkdocs.yml           |  0
 bfabric_scripts/{doc => docs}/changelog.md |  0
 noxfile.py                                 | 17 ++++++++++++++++-
 10 files changed, 16 insertions(+), 1 deletion(-)
 rename {docs => bfabric/docs}/changelog.md (100%)
 rename {docs => bfabric/docs}/contribute.md (100%)
 rename {docs => bfabric/docs}/entities.md (100%)
 rename {docs => bfabric/docs}/good_to_know.md (100%)
 rename {docs => bfabric/docs}/index.md (100%)
 rename {docs => bfabric/docs}/old/cheatsheet.md (100%)
 rename {docs => bfabric/docs}/old/faq.md (100%)
 rename mkdocs.yml => bfabric/mkdocs.yml (100%)
 rename bfabric_scripts/{doc => docs}/changelog.md (100%)

diff --git a/docs/changelog.md b/bfabric/docs/changelog.md
similarity index 100%
rename from docs/changelog.md
rename to bfabric/docs/changelog.md
diff --git a/docs/contribute.md b/bfabric/docs/contribute.md
similarity index 100%
rename from docs/contribute.md
rename to bfabric/docs/contribute.md
diff --git a/docs/entities.md b/bfabric/docs/entities.md
similarity index 100%
rename from docs/entities.md
rename to bfabric/docs/entities.md
diff --git a/docs/good_to_know.md b/bfabric/docs/good_to_know.md
similarity index 100%
rename from docs/good_to_know.md
rename to bfabric/docs/good_to_know.md
diff --git a/docs/index.md b/bfabric/docs/index.md
similarity index 100%
rename from docs/index.md
rename to bfabric/docs/index.md
diff --git a/docs/old/cheatsheet.md b/bfabric/docs/old/cheatsheet.md
similarity index 100%
rename from docs/old/cheatsheet.md
rename to bfabric/docs/old/cheatsheet.md
diff --git a/docs/old/faq.md b/bfabric/docs/old/faq.md
similarity index 100%
rename from docs/old/faq.md
rename to bfabric/docs/old/faq.md
diff --git a/mkdocs.yml b/bfabric/mkdocs.yml
similarity index 100%
rename from mkdocs.yml
rename to bfabric/mkdocs.yml
diff --git a/bfabric_scripts/doc/changelog.md b/bfabric_scripts/docs/changelog.md
similarity index 100%
rename from bfabric_scripts/doc/changelog.md
rename to bfabric_scripts/docs/changelog.md
diff --git a/noxfile.py b/noxfile.py
index 3def5899..ca86ce7a 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -1,4 +1,7 @@
+import os
 import shutil
+from collections.abc import Generator
+from contextlib import contextmanager
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -8,6 +11,17 @@
 nox.options.default_venv_backend = "uv"
 
 
+@contextmanager
+def chdir(path: Path) -> Generator[None, None, None]:
+    """Context manager to change directory."""
+    cwd = Path.cwd()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(cwd)
+
+
 @nox.session(python=["3.9", "3.11", "3.13"])
 def tests(session):
     session.install("./bfabric[test]", "-e", "./bfabric_scripts")
@@ -46,7 +60,8 @@ def docs(session):
     """Builds documentation for bfabricPy and app-runner and writes to site directory."""
     with TemporaryDirectory() as tmpdir:
         session.install("./bfabric[doc]")
-        session.run("mkdocs", "build", "-d", Path(tmpdir) / "build_bfabricpy")
+        with chdir("bfabric"):
+            session.run("mkdocs", "build", "-d", Path(tmpdir) / "build_bfabricpy")
 
         session.install("./bfabric_app_runner[doc]")
         session.run(

From 7d0c73c41e520553f637a5906d208cbd193585bc Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Tue, 18 Feb 2025 14:02:20 +0100
Subject: [PATCH 26/35] Fix problem due to __future__.annotations breaking
 cyclopts

---
 bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py | 1 -
 bfabric_app_runner/src/bfabric_app_runner/cli/app.py      | 2 --
 bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py    | 2 --
 bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py   | 2 --
 bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py  | 2 --
 bfabric_app_runner/src/bfabric_app_runner/cli/validate.py | 2 --
 6 files changed, 11 deletions(-)

diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py
index 94bfe48e..06b2996e 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py
@@ -16,7 +16,6 @@
 )
 
 package_version = importlib.metadata.version("bfabric_app_runner")
-
 app = cyclopts.App(
     help="Provides an entrypoint to app execution.\n\nFunctionality/API under active development!",
     version=package_version,
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
index 222c3617..0df04dca 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import importlib.metadata
 import importlib.resources
 from pathlib import Path
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py b/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py
index e21d55fd..57bd408d 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/chunk.py
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from pathlib import Path
 
 from bfabric import Bfabric
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py b/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py
index 282fc267..4f3008d0 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/inputs.py
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from pathlib import Path
 
 from bfabric import Bfabric
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py b/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py
index fe665d8c..7b714bca 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/outputs.py
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from pathlib import Path
 
 from rich.pretty import pprint
diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py b/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py
index 3fba6153..93637629 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/cli/validate.py
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from pathlib import Path
 
 import yaml

From a9e6ccf1a5b3238d33adf46f75847df01c606d83 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 19 Feb 2025 11:01:33 +0100
Subject: [PATCH 27/35] Update default behavior of
 CopyResourceSpec.update_existing to 'if_exists'

---
 bfabric_app_runner/docs/changelog.md                         | 1 +
 .../src/bfabric_app_runner/specs/outputs_spec.py             | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/bfabric_app_runner/docs/changelog.md b/bfabric_app_runner/docs/changelog.md
index 46d9661c..6f80efff 100644
--- a/bfabric_app_runner/docs/changelog.md
+++ b/bfabric_app_runner/docs/changelog.md
@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Changed
 
+- CopyResourceSpec.update_existing now defaults to `if_exists`.
 - Resolve workunit_ref to absolute path if it is a Path instance for CLI.
 
 ## \[0.0.15\] - 2025-02-06
diff --git a/bfabric_app_runner/src/bfabric_app_runner/specs/outputs_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/outputs_spec.py
index 8660e6fa..e7e32a98 100644
--- a/bfabric_app_runner/src/bfabric_app_runner/specs/outputs_spec.py
+++ b/bfabric_app_runner/src/bfabric_app_runner/specs/outputs_spec.py
@@ -27,7 +27,9 @@ class CopyResourceSpec(BaseModel):
     store_folder_path: Path | None = None
     """The storage folder will be determined by the default rule, but can be specified if needed."""
 
-    update_existing: UpdateExisting = UpdateExisting.NO
+    # TODO these need to be implemented properly (e.g. do not scp too early), and tested in integration tests
+    update_existing: UpdateExisting = UpdateExisting.IF_EXISTS
+
     protocol: Literal["scp"] = "scp"
 
 
@@ -35,6 +37,7 @@ class SaveDatasetSpec(BaseModel):
     model_config = ConfigDict(extra="forbid")
     type: Literal["bfabric_dataset"] = "bfabric_dataset"
 
+    # TODO this will currently fail if the workunit already has an output dataset -> needs to be handled as well
     local_path: Path
     separator: str
     name: str | None = None

From 561b4124576a85ec7b5ea0ee216e06e728db4d42 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 19 Feb 2025 11:11:56 +0100
Subject: [PATCH 28/35] create the release automatically

---
 .github/workflows/publish_release.yml | 43 ++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml
index 2a35682c..305a2ac6 100644
--- a/.github/workflows/publish_release.yml
+++ b/.github/workflows/publish_release.yml
@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       id-token: write
-      contents: write # permission to create tags
+      contents: write # permission to create tags and releases
     steps:
       - uses: actions/checkout@v4
       # Step: Determine the package that is being built
@@ -86,6 +86,47 @@ jobs:
           TAG_NAME="${{ env.PACKAGE }}/${{ env.VERSION }}"
           git tag -a "$TAG_NAME" -m "Release ${{ env.PACKAGE }} version ${{ env.VERSION }}"
           git push origin "$TAG_NAME"
+      - name: Extract changelog and create release
+        run: |
+          CHANGELOG_PATH="${{ env.PACKAGE }}/docs/changelog.md"
+          TAG_NAME="${{ env.PACKAGE }}/${{ env.VERSION }}"
+
+          if [ -f "$CHANGELOG_PATH" ]; then
+            # Extract the changelog section for the current version
+            # This assumes the changelog follows the keep-a-changelog format
+            CHANGELOG_CONTENT=$(awk -v ver="${{ env.VERSION }}" '
+              BEGIN { found=0; }
+              $0 ~ "^## \\[" ver "\\]" { found=1; next }
+              found && $0 ~ "^## \\[" { exit }
+              found { print }
+            ' "$CHANGELOG_PATH")
+
+            if [ -n "$CHANGELOG_CONTENT" ]; then
+              # Create GitHub release
+              gh release create "$TAG_NAME" \
+                --title "Release ${{ env.PACKAGE }} ${{ env.VERSION }}" \
+                --notes "$CHANGELOG_CONTENT" \
+                --target ${{ github.sha }} \
+                --draft
+            else
+              # Create release with basic notes if no changelog section is found
+              gh release create "$TAG_NAME" \
+                --title "Release ${{ env.PACKAGE }} ${{ env.VERSION }}" \
+                --notes "Release of ${{ env.PACKAGE }} version ${{ env.VERSION }}" \
+                --target ${{ github.sha }} \
+                --draft
+            fi
+          else
+            echo "Warning: Changelog file not found at $CHANGELOG_PATH"
+            # Create release with basic notes
+            gh release create "$TAG_NAME" \
+              --title "Release ${{ env.PACKAGE }} ${{ env.VERSION }}" \
+              --notes "Release of ${{ env.PACKAGE }} version ${{ env.VERSION }}" \
+              --target ${{ github.sha }} \
+              --draft
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Debug package info
         run: |
           echo "Built and published package: ${{ env.PACKAGE }}"

From ae39b307d4d813ffcf602a6f966d44f6dffb07e6 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 19 Feb 2025 11:12:14 +0100
Subject: [PATCH 29/35] bfabricPy 1.13.21

---
 bfabric/docs/changelog.md | 2 +-
 bfabric/pyproject.toml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bfabric/docs/changelog.md b/bfabric/docs/changelog.md
index 6591aef8..5349c026 100644
--- a/bfabric/docs/changelog.md
+++ b/bfabric/docs/changelog.md
@@ -8,7 +8,7 @@ Versioning currently follows `X.Y.Z` where
 - `Y` should be the current bfabric release
 - `Z` is increased for feature releases, that should not break the API
 
-## \[Unreleased\]
+## \[1.13.21\] - 2025-02-19
 
 ### Added
 
diff --git a/bfabric/pyproject.toml b/bfabric/pyproject.toml
index df26c2f5..c0883f91 100644
--- a/bfabric/pyproject.toml
+++ b/bfabric/pyproject.toml
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "bfabric"
 description = "Python client for the B-Fabric API"
-version = "1.13.20"
+version = "1.13.21"
 license = { text = "GPL-3.0" }
 authors = [
     { name = "Christian Panse", email = "cp@fgcz.ethz.ch" },

From a3d7df13adc9f75768ac4f9ab5bf5acaa24f19bf Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 19 Feb 2025 11:18:07 +0100
Subject: [PATCH 30/35] decouple testing of packages

---
 bfabric_scripts/pyproject.toml |  3 +++
 noxfile.py                     | 15 +++++++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/bfabric_scripts/pyproject.toml b/bfabric_scripts/pyproject.toml
index 99c6b880..21e06374 100644
--- a/bfabric_scripts/pyproject.toml
+++ b/bfabric_scripts/pyproject.toml
@@ -11,6 +11,9 @@ dependencies = [
     "bfabric==1.13.20"
 ]
 
+[project.optional-dependencies]
+test = ["pytest", "pytest-mock", "logot[pytest,loguru]"]
+
 [project.scripts]
 "bfabric_flask.py" = "bfabric_scripts.bfabric_flask:main"
 #bfabric_feeder_resource_autoQC="bfabric_scripts.bfabric_feeder_resource_autoQC:main"
diff --git a/noxfile.py b/noxfile.py
index ca86ce7a..fe005e12 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -23,17 +23,24 @@ def chdir(path: Path) -> Generator[None, None, None]:
 
 
 @nox.session(python=["3.9", "3.11", "3.13"])
-def tests(session):
-    session.install("./bfabric[test]", "-e", "./bfabric_scripts")
+def test_bfabric(session):
+    session.install("./bfabric[test]")
     session.run("uv", "pip", "list")
-    packages = ["tests/bfabric", "tests/bfabric_scripts"]
+    session.run("pytest", "--durations=50", "tests/bfabric")
+
+
+@nox.session(python=["3.9", "3.11", "3.13"])
+def test_bfabric_scripts(session):
+    session.install("-e", "./bfabric_scripts[test]")
+    session.run("uv", "pip", "list")
+    packages = ["tests/bfabric_scripts"]
     if session.python.split(".")[0] == "3" and int(session.python.split(".")[1]) >= 11:
         packages.append("tests/bfabric_cli")
     session.run("pytest", "--durations=50", *packages)
 
 
 @nox.session(python=["3.13"])
-def test_app_runner(session):
+def test_bfabric_app_runner(session):
     session.install("-e", "./bfabric")
     session.install("./bfabric_app_runner[test]")
     session.run("uv", "pip", "list")

From 710bc03356e68e637feb0ef6a17f775eb4d57791 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 19 Feb 2025 11:24:12 +0100
Subject: [PATCH 31/35] update the parsing of the changelog

---
 .github/workflows/publish_release.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml
index 305a2ac6..b93a2a20 100644
--- a/.github/workflows/publish_release.yml
+++ b/.github/workflows/publish_release.yml
@@ -88,6 +88,7 @@ jobs:
           git push origin "$TAG_NAME"
       - name: Extract changelog and create release
         run: |
+          set -x
           CHANGELOG_PATH="${{ env.PACKAGE }}/docs/changelog.md"
           TAG_NAME="${{ env.PACKAGE }}/${{ env.VERSION }}"
 
@@ -95,10 +96,10 @@ jobs:
             # Extract the changelog section for the current version
             # This assumes the changelog follows the keep-a-changelog format
             CHANGELOG_CONTENT=$(awk -v ver="${{ env.VERSION }}" '
-              BEGIN { found=0; }
-              $0 ~ "^## \\[" ver "\\]" { found=1; next }
-              found && $0 ~ "^## \\[" { exit }
-              found { print }
+            BEGIN { found=0; }
+            $0 ~ "^## \\\\[" ver "\\\\]" { found=1; next }
+            found && $0 ~ "^## \\\\[" { exit }
+            found { print }
             ' "$CHANGELOG_PATH")
 
             if [ -n "$CHANGELOG_CONTENT" ]; then

From f5d79dcef2b9d06e6dd9ab87927b51f3958b9922 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 19 Feb 2025 11:30:16 +0100
Subject: [PATCH 32/35] fix changelog merge

---
 .gitattributes                    | 1 +
 bfabric_scripts/docs/changelog.md | 9 +++------
 2 files changed, 4 insertions(+), 6 deletions(-)
 create mode 100644 .gitattributes

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000..23131993
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+changelog.md merge=manual
diff --git a/bfabric_scripts/docs/changelog.md b/bfabric_scripts/docs/changelog.md
index 8095c6a3..cc234b64 100644
--- a/bfabric_scripts/docs/changelog.md
+++ b/bfabric_scripts/docs/changelog.md
@@ -13,6 +13,9 @@ Versioning currently follows `X.Y.Z` where
 ### Added
 
 - `bfabric-cli dataset {upload, download, show}` to replace the old dataset-related scripts.
+- `bfabric-cli api update` command to update an existing entity
+- `bfabric-cli api create` command to create a new entity
+- `bfabric-cli api delete` command to delete an existing entity
 
 ## \[1.13.22\] - 2025-02-17
 
@@ -26,12 +29,6 @@ Versioning currently follows `X.Y.Z` where
 
 - Add missing default value for columns in `bfabric-cli api read`
 
-### Added
-
-- `bfabric-cli api update` command to update an existing entity
-- `bfabric-cli api create` command to create a new entity
-- `bfabric-cli api delete` command to delete an existing entity
-
 ## \[1.13.20\] - 2025-02-10
 
 ### Added

From b6fdba1c549bd6173cf00efdc04c42edda702264 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 19 Feb 2025 11:35:43 +0100
Subject: [PATCH 33/35] check if the version exists in changelog

---
 noxfile.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/noxfile.py b/noxfile.py
index fe005e12..90930f45 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -1,5 +1,7 @@
 import os
+import re
 import shutil
+import tomllib
 from collections.abc import Generator
 from contextlib import contextmanager
 from pathlib import Path
@@ -110,3 +112,52 @@ def licensecheck(session) -> None:
     # TODO is there a better way
     session.install("licensecheck")
     session.run("sh", "-c", "cd bfabric && licensecheck")
+
+
+def verify_changelog_version(session: nox.Session, package_dir: str) -> None:
+    """
+    Verify that the changelog contains an entry for the current version.
+
+    Args:
+        session: The nox session
+        package_dir: The package directory to check (e.g., 'bfabric', 'bfabric_scripts')
+
+    Raises:
+        nox.CommandFailed: If the changelog doesn't contain the current version
+    """
+    package_path = Path(package_dir)
+
+    # Read version from pyproject.toml
+    try:
+        with open(package_path / "pyproject.toml", "rb") as f:
+            pyproject = tomllib.load(f)
+            current_version = pyproject["project"]["version"]
+    except (FileNotFoundError, KeyError) as e:
+        session.error(f"Failed to read version from pyproject.toml: {e}")
+
+    # Read and check changelog
+    changelog_path = package_path / "docs" / "changelog.md"
+    try:
+        changelog_content = changelog_path.read_text()
+    except FileNotFoundError:
+        session.error(f"Changelog not found at {changelog_path}")
+
+    # Look for version header with escaped brackets
+    version_pattern = rf"## \\\[{re.escape(current_version)}\\\]"
+    if not re.search(version_pattern, changelog_content):
+        session.error(
+            f"{changelog_path} does not contain entry for version {current_version}.\n"
+            f"Expected to find a section starting with: ## \\[{current_version}\\]"
+        )
+
+    session.log(f"✓ {changelog_path} contains entry for version {current_version}")
+
+
+@nox.session
+def check_changelog(session: nox.Session):
+    """Check that changelog contains current version for all packages being released."""
+    # List of packages to check - could be made configurable
+    packages = ["bfabric", "bfabric_scripts", "bfabric_app_runner"]
+
+    for package in packages:
+        verify_changelog_version(session, package)

From 498c9331e480ffa8db614a6b9bd76270c9349f63 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 19 Feb 2025 11:36:02 +0100
Subject: [PATCH 34/35] bfabric-scripts 1.13.23

---
 bfabric_scripts/docs/changelog.md | 2 ++
 bfabric_scripts/pyproject.toml    | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/bfabric_scripts/docs/changelog.md b/bfabric_scripts/docs/changelog.md
index cc234b64..d4c5fc1f 100644
--- a/bfabric_scripts/docs/changelog.md
+++ b/bfabric_scripts/docs/changelog.md
@@ -10,6 +10,8 @@ Versioning currently follows `X.Y.Z` where
 
 ## \[Unreleased\]
 
+## \[1.13.23\] - 2025-02-19
+
 ### Added
 
 - `bfabric-cli dataset {upload, download, show}` to replace the old dataset-related scripts.
diff --git a/bfabric_scripts/pyproject.toml b/bfabric_scripts/pyproject.toml
index 21e06374..82bd891a 100644
--- a/bfabric_scripts/pyproject.toml
+++ b/bfabric_scripts/pyproject.toml
@@ -5,10 +5,10 @@ build-backend = "hatchling.build"
 [project]
 name = "bfabric_scripts"
 description = "Python command line scripts for the B-Fabric API"
-version = "1.13.22"
+version = "1.13.23"
 
 dependencies = [
-    "bfabric==1.13.20"
+    "bfabric==1.13.21"
 ]
 
 [project.optional-dependencies]

From 2d2f25074a07ee01a6ff44307987026d47cb98c4 Mon Sep 17 00:00:00 2001
From: Leonardo Schwarz <leonardo.schwarz@fgcz.ethz.ch>
Date: Wed, 19 Feb 2025 11:38:41 +0100
Subject: [PATCH 35/35] update to 3.11 for tomllib

---
 .github/workflows/run_unit_tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run_unit_tests.yml b/.github/workflows/run_unit_tests.yml
index 45a8bd57..8528d9f2 100644
--- a/.github/workflows/run_unit_tests.yml
+++ b/.github/workflows/run_unit_tests.yml
@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.11
       - name: Install nox
         run: pip install nox uv
       - name: Run checks
@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.11
       - name: Install nox
         run: pip install nox uv
       - name: Check code with ruff