diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
index f6fe391d..644310dc 100644
--- a/.github/workflows/test-integration.yml
+++ b/.github/workflows/test-integration.yml
@@ -92,6 +92,7 @@ jobs:
- name: Create artifact name
id: artifact-name
+ if: always()
run: |
if [ "${{ matrix.jupyterhub }}" = "4.1.5" ]; then
jhub_suffix="4x"
diff --git a/environment-dev.yml b/environment-dev.yml
index 0186e2ce..93d76968 100644
--- a/environment-dev.yml
+++ b/environment-dev.yml
@@ -22,3 +22,5 @@ dependencies:
- cachetools
- structlog
- gradio
+ - gitpython
+ - conda-project=0.4.2
diff --git a/jhub_apps/service/app_from_git.py b/jhub_apps/service/app_from_git.py
new file mode 100644
index 00000000..6dd7dda7
--- /dev/null
+++ b/jhub_apps/service/app_from_git.py
@@ -0,0 +1,125 @@
+import os
+import pathlib
+import tempfile
+from pathlib import Path
+
+import git
+from fastapi import HTTPException, status
+from pydantic import ValidationError
+
+from jhub_apps.service.models import Repository, JHubAppConfig
+from jhub_apps.service.utils import logger, encode_file_to_data_url
+
+
+def _clone_repo(repository: Repository, temp_dir):
+ """Clone repository to the given tem_dir"""
+ try:
+ logger.info("Trying to clone repository", repo_url=repository.url)
+ git.Repo.clone_from(repository.url, temp_dir, depth=1, branch=repository.ref)
+ except Exception as e:
+ message = f"Repository clone failed: {repository.url}"
+ logger.error(message, repo_url=repository.url)
+ logger.error(e)
+ raise HTTPException(
+ detail=message,
+ status_code=status.HTTP_400_BAD_REQUEST,
+ )
+
+
+def _get_app_configuration_from_git(
+ repository: Repository
+) -> JHubAppConfig:
+ """Clones the git directory into a temporary path and extracts all the metadata
+ about the app from conda-project's config yaml.
+ """
+ with tempfile.TemporaryDirectory() as temp_dir:
+ _clone_repo(repository, temp_dir)
+ _check_conda_project_config_directory_exists(repository, temp_dir)
+ conda_project_yaml = _get_conda_project_config_yaml(temp_dir)
+ jhub_apps_config_dict = _extract_jhub_apps_config_from_conda_project_config(conda_project_yaml)
+ app_config = _load_jhub_app_config_to_pydantic_model(
+ jhub_apps_config_dict,
+ repository,
+ temp_dir
+ )
+ return app_config
+
+
+def _load_jhub_app_config_to_pydantic_model(
+ jhub_apps_config_dict: dict, repository: Repository, temp_dir: str
+):
+ """Load the parsed jhub-apps config into pydantic model for validation"""
+ thumbnail_base64 = ""
+ thumbnail_path_from_config = jhub_apps_config_dict.get("thumbnail_path")
+ if thumbnail_path_from_config:
+ thumbnail_path = Path(os.path.join(temp_dir, thumbnail_path_from_config))
+ thumbnail_base64 = encode_file_to_data_url(
+ filename=thumbnail_path.name, file_contents=thumbnail_path.read_bytes()
+ )
+ try:
+ # Load YAML content into the Pydantic model
+ app_config = JHubAppConfig(**{
+ **jhub_apps_config_dict,
+ "repository": repository,
+ "thumbnail": thumbnail_base64,
+ "env": jhub_apps_config_dict.get("environment", {})
+ })
+ except ValidationError as e:
+ message = f"Validation error: {e}"
+ logger.error(message)
+ raise HTTPException(
+ detail=message,
+ status_code=status.HTTP_400_BAD_REQUEST,
+ )
+ return app_config
+
+
+def _extract_jhub_apps_config_from_conda_project_config(conda_project_yaml):
+ """Extracts jhub-apps app config from conda project yaml's config"""
+ jhub_apps_variables = {
+ k.split("JHUB_APP_CONFIG_")[-1]: v for k, v in conda_project_yaml.variables.items()
+ if k.startswith("JHUB_APP_CONFIG_")
+ }
+ environment_variables = {
+ k: v for k, v in conda_project_yaml.variables.items()
+ if not k.startswith("JHUB_APP_CONFIG_")
+ }
+ return {
+ **jhub_apps_variables,
+ "environment": environment_variables
+ }
+
+
+def _get_conda_project_config_yaml(directory: str):
+ """Given the directory, get conda project config object"""
+ # Moving this to top level import causes this problem:
+ # https://github.com/jupyter/jupyter_events/issues/99
+ from conda_project import CondaProject, CondaProjectError
+ from conda_project.project_file import CondaProjectYaml
+ try:
+ conda_project = CondaProject(directory)
+ # This is a private attribute, ideally we shouldn't access it,
+ # but I haven't found an alternative way to get this
+ conda_project_yaml: CondaProjectYaml = conda_project._project_file
+ except CondaProjectError as e:
+ message = "Invalid conda-project"
+ logger.error(message)
+ logger.exception(e)
+ raise HTTPException(
+ detail=message,
+ status_code=status.HTTP_400_BAD_REQUEST,
+ )
+ return conda_project_yaml
+
+
+def _check_conda_project_config_directory_exists(repository: Repository, temp_dir: str):
+ """Check if the conda project config directory provided by the user exists"""
+ temp_dir_path = pathlib.Path(temp_dir)
+ conda_project_dir = temp_dir_path / repository.config_directory
+ if not conda_project_dir.exists():
+ message = f"Path '{repository.config_directory}' doesn't exists in the repository."
+ logger.error(message, repo_url=repository.url)
+ raise HTTPException(
+ detail=message,
+ status_code=status.HTTP_400_BAD_REQUEST,
+ )
diff --git a/jhub_apps/service/models.py b/jhub_apps/service/models.py
index 796b8902..42442b1f 100644
--- a/jhub_apps/service/models.py
+++ b/jhub_apps/service/models.py
@@ -55,23 +55,34 @@ class HubApiError(BaseModel):
detail: HubResponse
-class UserOptions(BaseModel):
- jhub_app: bool
+class Repository(BaseModel):
+ url: str
+ config_directory: str = "."
+ # git ref
+ ref: str = "main"
+
+
+class JHubAppConfig(BaseModel):
display_name: str
description: str
thumbnail: str = None
filepath: typing.Optional[str] = str()
framework: str = "panel"
custom_command: typing.Optional[str] = str()
- conda_env: typing.Optional[str] = str()
- # Environment variables
- env: typing.Optional[dict] = dict()
- profile: typing.Optional[str] = str()
# Make app available to public (unauthenticated Hub users)
public: typing.Optional[bool] = False
# Keep app alive, even when there is no activity
# So that it's not killed by idle culler
keep_alive: typing.Optional[bool] = False
+ # Environment variables
+ env: typing.Optional[dict] = dict()
+ repository: typing.Optional[Repository] = None
+
+
+class UserOptions(JHubAppConfig):
+ jhub_app: bool
+ conda_env: typing.Optional[str] = str()
+ profile: typing.Optional[str] = str()
share_with: typing.Optional[SharePermissions] = None
diff --git a/jhub_apps/service/routes.py b/jhub_apps/service/routes.py
index 424b828e..8b67e59f 100644
--- a/jhub_apps/service/routes.py
+++ b/jhub_apps/service/routes.py
@@ -28,6 +28,8 @@
HubApiError,
ServerCreation,
User,
+ Repository,
+ JHubAppConfig,
)
from jhub_apps.service.security import get_current_user
from jhub_apps.service.utils import (
@@ -37,6 +39,7 @@
get_thumbnail_data_url,
get_shared_servers,
)
+from jhub_apps.service.app_from_git import _get_app_configuration_from_git
from jhub_apps.spawner.types import FRAMEWORKS
from jhub_apps.version import get_version
@@ -271,6 +274,24 @@ async def hub_services(user: User = Depends(get_current_user)):
return hub_client.get_services()
+@router.post("/app-config-from-git/",)
+async def app_from_git(
+ repo: Repository,
+ user: User = Depends(get_current_user)
+) -> JHubAppConfig:
+ """
+ ## Fetches jhub-apps application configuration from a git repository.
+
+ Note: This endpoint is kept as POST intentionally because the client is
+ requesting the server to process some data, in this case, to fetch
+ a repository, read its conda project config, and return specific values,
+ which is a processing action.
+ """
+ logger.info("Getting app configuration from git repository")
+ response = _get_app_configuration_from_git(repo)
+ return response
+
+
@router.get("/")
@router.get("/status")
async def status_endpoint():
diff --git a/jhub_apps/spawner/spawner_creation.py b/jhub_apps/spawner/spawner_creation.py
index 9d3bc4cd..afa2ddac 100644
--- a/jhub_apps/spawner/spawner_creation.py
+++ b/jhub_apps/spawner/spawner_creation.py
@@ -1,3 +1,5 @@
+import uuid
+
import structlog
from jhub_apps.spawner.utils import get_origin_host
@@ -56,6 +58,18 @@ def get_args(self):
command = Command(args=GENERIC_ARGS + custom_cmd.split())
else:
command: Command = COMMANDS.get(framework)
+
+ repository = self.user_options.get("repository")
+ if repository:
+ logger.info(f"repository specified: {repository}")
+ # The repository will be cloned during spawn time to
+ # deploy the app from the repository.
+ command.args.extend([
+ f"--repo={repository.get('url')}",
+ f"--repofolder=/tmp/{self.name}-{uuid.uuid4().hex[:6]}",
+ f"--repobranch={repository.get('ref')}"
+ ])
+
command_args = command.get_substituted_args(
python_exec=self.config.JAppsConfig.python_exec,
filepath=app_filepath,
diff --git a/jhub_apps/tests/common/constants.py b/jhub_apps/tests/common/constants.py
index af6ae2c2..575afe62 100644
--- a/jhub_apps/tests/common/constants.py
+++ b/jhub_apps/tests/common/constants.py
@@ -4,3 +4,6 @@
MOCK_USER.name = "jovyan"
JUPYTERHUB_HOSTNAME = "127.0.0.1:8000"
+JUPYTERHUB_USERNAME = "admin"
+JUPYTERHUB_PASSWORD = "admin"
+JHUB_APPS_API_BASE_URL = f"http://{JUPYTERHUB_HOSTNAME}/services/japps"
diff --git a/jhub_apps/tests/tests_e2e/test_api.py b/jhub_apps/tests/tests_e2e/test_api.py
index 37a88daa..fdaa8b27 100644
--- a/jhub_apps/tests/tests_e2e/test_api.py
+++ b/jhub_apps/tests/tests_e2e/test_api.py
@@ -1,4 +1,110 @@
+import hashlib
+
+import pytest
+
+from jhub_apps.service.models import Repository, UserOptions, ServerCreation
+from jhub_apps.tests.common.constants import JHUB_APPS_API_BASE_URL, JUPYTERHUB_HOSTNAME
+from jhub_apps.tests.tests_e2e.utils import get_jhub_apps_session, fetch_url_until_title_found
+
+EXAMPLE_TEST_REPO = "https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git"
+
+
def test_api_status(client):
response = client.get("/status")
assert response.status_code == 200
assert set(response.json().keys()) == {"version", "status"}
+
+
+def test_app_config_from_git_api(
+ client,
+):
+ response = client.post(
+ '/app-config-from-git/',
+ json={
+ "url": EXAMPLE_TEST_REPO,
+ "config_directory": ".",
+ "ref": "main"
+ }
+ )
+ assert response.status_code == 200
+ response_json = response.json()
+ assert response_json
+ assert set(response_json.keys()) == {
+ "display_name", "description", "framework", "filepath",
+ "env", "keep_alive", "public", "thumbnail",
+ "custom_command", "repository"
+ }
+ assert response_json["display_name"] == "My Panel App (Git)"
+ assert response_json["description"] == "This is a panel app created from git repository"
+ assert response_json["framework"] == "panel"
+ assert response_json["filepath"] == "panel_basic.py"
+ assert response_json["env"] == {
+ "SOMETHING_FOO": "bar",
+ "SOMETHING_BAR": "beta",
+ }
+ assert response_json["keep_alive"] is False
+ assert response_json["public"] is False
+
+ assert isinstance(response_json["thumbnail"], str)
+ expected_thumbnail_sha = "a8104b2482360eee525dc696dafcd2a17864687891dc1b6c9e21520518a5ea89"
+ assert hashlib.sha256(response_json["thumbnail"].encode('utf-8')).hexdigest() == expected_thumbnail_sha
+
+
+@pytest.mark.parametrize("repo_url, config_directory, response_status_code,detail", [
+ (EXAMPLE_TEST_REPO, "non-existent-path", 400,
+ "Path 'non-existent-path' doesn't exists in the repository."),
+ ("http://invalid-repo/", ".", 400,
+ "Repository clone failed: http://invalid-repo/"),
+])
+def test_app_config_from_git_api_invalid(
+ client,
+ repo_url,
+ config_directory,
+ response_status_code,
+ detail
+):
+ response = client.post(
+ '/app-config-from-git/',
+ json={
+ "url": repo_url,
+ "config_directory": config_directory,
+ "ref": "main"
+ }
+ )
+ assert response.status_code == response_status_code
+ response_json = response.json()
+ assert "detail" in response_json
+ assert response_json["detail"] == detail
+
+
+def test_create_server_with_git_repository():
+ user_options = UserOptions(
+ jhub_app=True,
+ display_name="Test Application",
+ description="App description",
+ framework="panel",
+ thumbnail="data:image/png;base64,ZHVtbXkgaW1hZ2UgZGF0YQ==",
+ filepath="panel_basic.py",
+ repository=Repository(
+ url="https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git",
+ )
+ )
+ files = {"thumbnail": ("test.png", b"dummy image data", "image/png")}
+ server_data = ServerCreation(
+ servername="test server from git repo",
+ user_options=user_options
+ )
+ data = {"data": server_data.model_dump_json()}
+ session = get_jhub_apps_session()
+ response = session.post(
+ f"{JHUB_APPS_API_BASE_URL}/server",
+ verify=False,
+ data=data,
+ files=files
+ )
+ assert response.status_code == 200
+ server_name = response.json()[-1]
+ created_app_url = f"http://{JUPYTERHUB_HOSTNAME}/user/admin/{server_name}/"
+ fetch_url_until_title_found(
+ session, url=created_app_url, expected_title="Panel Test App from Git Repository"
+ )
diff --git a/jhub_apps/tests/tests_e2e/test_integration.py b/jhub_apps/tests/tests_e2e/test_integration.py
index 0a11b047..30c970fe 100644
--- a/jhub_apps/tests/tests_e2e/test_integration.py
+++ b/jhub_apps/tests/tests_e2e/test_integration.py
@@ -148,8 +148,6 @@ def sign_in_and_authorize(page, username, password):
page.get_by_label("Password:").fill(password)
logger.info("Pressing Sign in button")
page.get_by_role("button", name="Sign in").click()
- logger.info("Click Authorize button")
- page.get_by_role("button", name="Authorize").click()
def sign_out(page):
diff --git a/jhub_apps/tests/tests_e2e/utils.py b/jhub_apps/tests/tests_e2e/utils.py
new file mode 100644
index 00000000..3ca4225d
--- /dev/null
+++ b/jhub_apps/tests/tests_e2e/utils.py
@@ -0,0 +1,60 @@
+import time
+
+import requests
+
+from jhub_apps.tests.common.constants import JUPYTERHUB_HOSTNAME, JUPYTERHUB_USERNAME, JUPYTERHUB_PASSWORD
+
+
+def get_jhub_apps_session():
+ """Get jhub-apps session with authenticated cookies to be able to call jhub-apps API"""
+ session = requests.Session()
+ session.cookies.clear()
+ try:
+ response = session.get(
+ f"http://{JUPYTERHUB_HOSTNAME}/hub/login", verify=False
+ )
+ response.raise_for_status()
+ auth_data = {
+ "_xsrf": session.cookies['_xsrf'],
+ "username": JUPYTERHUB_USERNAME,
+ "password": JUPYTERHUB_PASSWORD,
+ }
+ response = session.post(
+ f"http://{JUPYTERHUB_HOSTNAME}/hub/login?next=",
+ headers={"Content-Type": "application/x-www-form-urlencoded"},
+ data=auth_data,
+ verify=False,
+ )
+ response.raise_for_status()
+
+ except requests.RequestException as e:
+ raise ValueError(f"An error occurred during authentication: {e}")
+
+ response_login = session.get(
+ f"http://{JUPYTERHUB_HOSTNAME}/services/japps/jhub-login",
+ )
+ response_login.raise_for_status()
+ response_user = session.get(
+ f"http://{JUPYTERHUB_HOSTNAME}/services/japps/user",
+ verify=False
+ )
+ response_user.raise_for_status()
+ return session
+
+
+def fetch_url_until_title_found(
+ session, url, expected_title, timeout=10, interval=2
+):
+ """Fetches url until the expected title is found."""
+ start_time = time.time()
+ while True:
+ try:
+ response = session.get(url)
+ assert response.status_code == 200
+ if expected_title in str(response.content):
+ return
+ except (requests.RequestException, ValueError, AssertionError) as e:
+ time_elapsed = time.time() - start_time
+ if time_elapsed > timeout:
+ raise TimeoutError(f"Failed to get the title {expected_title} within {timeout} seconds") from e
+ time.sleep(interval)
diff --git a/jhub_apps/tests/tests_unit/test_api.py b/jhub_apps/tests/tests_unit/test_api.py
index 277a8f96..eea18882 100644
--- a/jhub_apps/tests/tests_unit/test_api.py
+++ b/jhub_apps/tests/tests_unit/test_api.py
@@ -5,6 +5,7 @@
import pytest
from jhub_apps.hub_client.hub_client import HubClient
+from jhub_apps.service.models import UserOptions, ServerCreation, Repository
from jhub_apps.service.utils import get_shared_servers
from jhub_apps.spawner.types import FRAMEWORKS
from jhub_apps.tests.common.constants import MOCK_USER
@@ -224,3 +225,32 @@ def test_open_api_docs(client):
assert response.status_code == 200
rjson = response.json()
assert rjson['info']['version']
+
+
+@patch.object(HubClient, "create_server")
+def test_create_server_with_git_repository(
+ hub_create_server,
+ client,
+):
+ user_options = UserOptions(
+ jhub_app=True,
+ display_name="Test Application",
+ description="App description",
+ framework="panel",
+ thumbnail="data:image/png;base64,ZHVtbXkgaW1hZ2UgZGF0YQ==",
+ filepath="panel_basic.py",
+ repository=Repository(
+ url="https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git",
+ )
+ )
+ server_data = ServerCreation(servername="test server", user_options=user_options)
+ files = {"thumbnail": ("test.png", b"dummy image data", "image/png")}
+ data = {"data": server_data.model_dump_json()}
+ hub_create_server.return_value = (201, 'test-server-abcdef')
+ response = client.post("/server", data=data, files=files)
+ assert response.status_code == 200
+ assert response.json() == [201, 'test-server-abcdef']
+ hub_create_server.assert_called_once_with(
+ username="jovyan", servername=server_data.servername,
+ user_options=user_options
+ )
diff --git a/jhub_apps/tests/tests_unit/test_app_from_git.py b/jhub_apps/tests/tests_unit/test_app_from_git.py
new file mode 100644
index 00000000..747dcc3b
--- /dev/null
+++ b/jhub_apps/tests/tests_unit/test_app_from_git.py
@@ -0,0 +1,31 @@
+from unittest.mock import Mock
+
+from jhub_apps.service.app_from_git import _extract_jhub_apps_config_from_conda_project_config
+
+
+def test_extract_jhub_apps_config_from_conda_project_config():
+ conda_project_yaml = Mock(variables={
+ "JHUB_APP_CONFIG_name": "My Panel App (Git)",
+ "JHUB_APP_CONFIG_description": "This is a panel app created from git repository",
+ "JHUB_APP_CONFIG_framework": "panel",
+ "JHUB_APP_CONFIG_filepath": "panel_basic.py",
+ "JHUB_APP_CONFIG_keep_alive": "false",
+ "JHUB_APP_CONFIG_public": "false",
+ "JHUB_APP_CONFIG_thumbnail_path": "panel.png",
+ "SOMETHING_FOO": "bar",
+ "SOMETHING_BAR": "beta",
+ })
+ jhub_apps_config = _extract_jhub_apps_config_from_conda_project_config(conda_project_yaml)
+ assert jhub_apps_config == {
+ "name": "My Panel App (Git)",
+ "description": "This is a panel app created from git repository",
+ "framework": "panel",
+ "filepath": "panel_basic.py",
+ "keep_alive": "false",
+ "public": "false",
+ "thumbnail_path": "panel.png",
+ "environment": {
+ "SOMETHING_FOO": "bar",
+ "SOMETHING_BAR": "beta",
+ }
+ }
diff --git a/jupyterhub_config.py b/jupyterhub_config.py
index 84cbcc7e..8dcf3f44 100644
--- a/jupyterhub_config.py
+++ b/jupyterhub_config.py
@@ -19,7 +19,7 @@
c.JAppsConfig.service_workers = 1
c.JupyterHub.default_url = "/hub/home"
-c = install_jhub_apps(c, spawner_to_subclass=SimpleLocalProcessSpawner)
+c = install_jhub_apps(c, spawner_to_subclass=SimpleLocalProcessSpawner, oauth_no_confirm=True)
c.JupyterHub.template_paths = theme_template_paths
diff --git a/pyproject.toml b/pyproject.toml
index 9df7e10d..5066198a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,6 +28,10 @@ dependencies = [
"cachetools",
"structlog",
"PyJWT",
+ "GitPython",
+ # pinning to avoid unexpected changes in spec causing
+ # unexpected breakage
+ "conda-project==0.4.2"
]
dynamic = ["version"]