From 2811931d69623604a8d666feec0ea1e9c2fa2ba5 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Tue, 27 Aug 2024 10:33:05 +0300 Subject: [PATCH] Support path to certificate files to be specified as a downloadable URL --- clearml/storage/helper.py | 25 ++ .../clearml_agent/clearml_colab_agent.ipynb | 364 +++++++++--------- 2 files changed, 207 insertions(+), 182 deletions(-) diff --git a/clearml/storage/helper.py b/clearml/storage/helper.py index a2928e03..a81f4da0 100644 --- a/clearml/storage/helper.py +++ b/clearml/storage/helper.py @@ -55,6 +55,7 @@ class DownloadError(Exception): @six.add_metaclass(ABCMeta) class _Driver(object): + _certs_cache_context = "certs" _file_server_hosts = None @classmethod @@ -116,6 +117,28 @@ def get_file_server_hosts(cls): cls._file_server_hosts = hosts return cls._file_server_hosts + @classmethod + def download_cert(cls, cert_url): + # import here to avoid circular imports + from .manager import StorageManager + + cls.get_logger().info("Attempting to download remote certificate '{}'".format(cert_url)) + potential_exception = None + downloaded_verify = None + try: + downloaded_verify = StorageManager.get_local_copy(cert_url, cache_context=cls._certs_cache_context) + except Exception as e: + potential_exception = e + if not downloaded_verify: + cls.get_logger().error( + "Failed downloading remote certificate '{}'{}".format( + cert_url, "Error is: {}".format(potential_exception) if potential_exception else "" + ) + ) + else: + cls.get_logger().info("Successfully downloaded remote certificate '{}'".format(cert_url)) + return downloaded_verify + class _HttpDriver(_Driver): """ LibCloud http/https adapter (simple, enough for now) """ @@ -447,6 +470,8 @@ def __init__(self, name, cfg): # True is a non-documented value for boto3, use None instead (which means verify) print("Using boto3 verify=None instead of true") verify = None + elif isinstance(verify, str) and not os.path.exists(verify) and verify.split("://")[0] in driver_schemes: + verify = _Boto3Driver.download_cert(verify) # boto3 client creation isn't thread-safe (client itself is) with self._creation_lock: diff --git a/examples/clearml_agent/clearml_colab_agent.ipynb b/examples/clearml_agent/clearml_colab_agent.ipynb index 3d087350..f6f6c807 100644 --- a/examples/clearml_agent/clearml_colab_agent.ipynb +++ b/examples/clearml_agent/clearml_colab_agent.ipynb @@ -1,185 +1,185 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "view-in-github" - }, - "source": [ - "\"Open" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mC3A4rio6Y--" - }, - "source": [ - "# Google Colab Used to Launch ClearML Agent\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EPYjmFLy1zuC" - }, - "source": [ - "## Step 1: Install all necessary packages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "DwFC3fL8JAP3" - }, - "outputs": [], - "source": [ - "# If you don't have ClearML and ClearML Agent installed then uncomment these line\n", - "# !pip install clearml\n", - "# !pip install clearml-agent" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gb41kM8i1-7T" - }, - "source": [ - "## Step 2: Export this environment variable\n", - "\n", - "This environment variable makes Matplotlib work in headless mode, so it won't output graphs to the screen" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "3-Bm4811VMLK" - }, - "outputs": [], - "source": [ - "! export MPLBACKEND=TkAg" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Step 2b (OPTIONAL): Enter your github credentials (only for private repositories)\n", - "In order for the agent to pull your code, it needs access to your repositories. If these are private, you'll have to supply the agent with github credentials to log in. Github/Bitbucket will no longer let you log in using username/password combinations. Instead, you have to use a personal token, read more about it [here for Github](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and [here for Bitbucket](https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/)\n", - "\n", - "We can let the agent know which credentials to use by setting the following env variables" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#@title Insert your Git Credentials\n", - "\n", - "import os\n", - "os.environ[\"CLEARML_AGENT_GIT_USER\"] = \"username-goes-here\" #@param {type:\"string\"}\n", - "os.environ[\"CLEARML_AGENT_GIT_PASS\"] = \"git-personal-access-token\" #@param {type:\"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "chuUzBaU2NyB" - }, - "source": [ - "## Step 3: Create new credentials: \n", - "1. Go to your [ClearML WebApp **Settings**](https://app.clear.ml/settings/workspace-configuration). \n", - "1. Under the **WORKSPACES** section, go to **App Credentials**, and click **+ Create new credentials**\n", - "1. Copy your credentials" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1Kloxwfj0Vnq" - }, - "source": [ - "## Step 4: Set your ClearML Credentials\n", - "\n", - "Insert the credentials you created in Step 3. \n", - "\n", - "If you aren't using the ClearML hosted server, make sure to modify the server variables. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "YBPdmP4sJHnQ" - }, - "outputs": [], - "source": [ - "#@title Insert your own Credentials\n", - "\n", - "from clearml import Task\n", - "\n", - "web_server = 'https://app.clear.ml'#@param {type:\"string\"} \n", - "api_server = 'https://api.clear.ml'#@param {type:\"string\"} \n", - "files_server = 'https://files.clear.ml'#@param {type:\"string\"}\n", - "access_key = ''#@param {type:\"string\"}\n", - "secret_key = ''#@param {type:\"string\"}\n", - "\n", - "Task.set_credentials(web_host=web_server,\n", - " api_host=api_server,\n", - " files_host=files_server,\n", - " key=access_key,\n", - " secret=secret_key\n", - " )\n", - " \n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Er3HUBty4m7i" - }, - "source": [ - "## Step 4: Run clearml-agent" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "QcczeU7OJ9G-" - }, - "outputs": [], - "source": [ - "!clearml-agent daemon --queue default" - ] - } - ], - "metadata": { - "colab": { - "authorship_tag": "ABX9TyMKeTOHrlQb98s9lDHxWFts", - "collapsed_sections": [], - "include_colab_link": true, - "name": "clearml_colab_agent.ipynb", - "provenance": [], - "toc_visible": true - }, - "interpreter": { - "hash": "d75e902da2bbfe9f41879fcf2334f5819447e02a7f656a079df344fef4e78809" - }, - "kernelspec": { - "display_name": "Python 3.7.12 64-bit ('.env')", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "view-in-github" + }, + "source": [ + "\"Open" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "markdown", + "metadata": { + "id": "mC3A4rio6Y--" + }, + "source": [ + "# Google Colab Used to Launch ClearML Agent\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EPYjmFLy1zuC" + }, + "source": [ + "## Step 1: Install all necessary packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DwFC3fL8JAP3" + }, + "outputs": [], + "source": [ + "# If you don't have ClearML and ClearML Agent installed then uncomment these line\n", + "# !pip install clearml\n", + "# !pip install clearml-agent" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gb41kM8i1-7T" + }, + "source": [ + "## Step 2: Export this environment variable\n", + "\n", + "This environment variable makes Matplotlib work in headless mode, so it won't output graphs to the screen" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3-Bm4811VMLK" + }, + "outputs": [], + "source": [ + "! export MPLBACKEND=TkAg" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 2b (OPTIONAL): Enter your github credentials (only for private repositories)\n", + "In order for the agent to pull your code, it needs access to your repositories. If these are private, you'll have to supply the agent with github credentials to log in. Github/Bitbucket will no longer let you log in using username/password combinations. Instead, you have to use a personal token, read more about it [here for Github](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and [here for Bitbucket](https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/)\n", + "\n", + "We can let the agent know which credentials to use by setting the following env variables" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#@title Insert your Git Credentials\n", + "\n", + "import os\n", + "os.environ[\"CLEARML_AGENT_GIT_USER\"] = \"username-goes-here\" #@param {type:\"string\"}\n", + "os.environ[\"CLEARML_AGENT_GIT_PASS\"] = \"git-personal-access-token\" #@param {type:\"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "chuUzBaU2NyB" + }, + "source": [ + "## Step 3: Create new credentials: \n", + "1. Go to your [ClearML WebApp **Settings**](https://app.clear.ml/settings/workspace-configuration). \n", + "1. Under the **WORKSPACES** section, go to **App Credentials**, and click **+ Create new credentials**\n", + "1. Copy your credentials" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1Kloxwfj0Vnq" + }, + "source": [ + "## Step 4: Set your ClearML Credentials\n", + "\n", + "Insert the credentials you created in Step 3. \n", + "\n", + "If you aren't using the ClearML hosted server, make sure to modify the server variables. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YBPdmP4sJHnQ" + }, + "outputs": [], + "source": [ + "#@title Insert your own Credentials\n", + "\n", + "from clearml import Task\n", + "\n", + "web_server = 'https://app.clear.ml'#@param {type:\"string\"} \n", + "api_server = 'https://api.clear.ml'#@param {type:\"string\"} \n", + "files_server = 'https://files.clear.ml'#@param {type:\"string\"}\n", + "access_key = ''#@param {type:\"string\"}\n", + "secret_key = ''#@param {type:\"string\"}\n", + "\n", + "Task.set_credentials(web_host=web_server,\n", + " api_host=api_server,\n", + " files_host=files_server,\n", + " key=access_key,\n", + " secret=secret_key\n", + " )\n", + " \n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Er3HUBty4m7i" + }, + "source": [ + "## Step 4: Run clearml-agent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QcczeU7OJ9G-" + }, + "outputs": [], + "source": [ + "!clearml-agent daemon --queue default" + ] + } + ], + "metadata": { + "colab": { + "authorship_tag": "ABX9TyMKeTOHrlQb98s9lDHxWFts", + "collapsed_sections": [], + "include_colab_link": true, + "name": "clearml_colab_agent.ipynb", + "provenance": [], + "toc_visible": true + }, + "interpreter": { + "hash": "d75e902da2bbfe9f41879fcf2334f5819447e02a7f656a079df344fef4e78809" + }, + "kernelspec": { + "display_name": "Python 3.7.12 64-bit ('.env')", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 0 }