Merge branch 'main' into patch-1

oracle · Feb 5, 2025 · dc4e185 · dc4e185
2 parents 00042ad + 7e9fc1b
commit dc4e185
Show file tree

Hide file tree

Showing 104 changed files with 7,787 additions and 1,204 deletions.
diff --git a/.github/workflows/run-forecast-unit-tests.yml b/.github/workflows/run-forecast-unit-tests.yml
@@ -56,6 +56,6 @@ jobs:
           $CONDA/bin/conda init
           source /home/runner/.bashrc
           pip install -r test-requirements-operators.txt
-          pip install "oracle-automlx[forecasting]>=24.4.0"
+          pip install "oracle-automlx[forecasting]>=24.4.1"
           pip install pandas>=2.2.0
           python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast
diff --git a/.gitignore b/.gitignore
@@ -86,7 +86,8 @@ celerybeat-schedule
 *.sage.py
 
 # dotenv
-.env
+.env*
+run_ads.sh
 
 # virtualenv
 .venv

diff --git a/README-development.md b/README-development.md
@@ -1,15 +1,41 @@
+<!-- TOC -->
 # Summary
 
 The Oracle Accelerated Data Science (ADS) SDK used by data scientists and analysts for
 data exploration and experimental machine learning to democratize machine learning and
-analytics by providing easy-to-use, performant, and user friendly tools that
+analytics by providing easy-to-use, 
+performant, and user friendly tools that
 brings together the best of data science practices.
 
 The ADS SDK helps you connect to different data sources, perform exploratory data analysis,
 data visualization, feature engineering, model training, model evaluation, and
 model interpretation. ADS also allows you to connect to the model catalog to save and load
 models to and from the catalog.
 
+- [Summary](#summary)
+  - [Documentation](#documentation)
+  - [Get Support](#get-support)
+  - [Getting started](#getting-started)
+    - [Step 1: Create a conda environment](#step-1-create-a-conda-environment)
+    - [Step 2: Activate your environment](#step-2-activate-your-environment)
+    - [Step 3: Clone ADS and install dependencies](#step-3-clone-ads-and-install-dependencies)
+    - [Step 4: Setup configuration files](#step-4-setup-configuration-files)
+    - [Step 5: Versioning and generation the wheel](#step-5-versioning-and-generation-the-wheel)
+  - [Running tests](#running-tests)
+    - [Running default setup tests](#running-default-setup-tests)
+    - [Running all unit tests](#running-all-unit-tests)
+    - [Running integration tests](#running-integration-tests)
+    - [Running opctl integration tests](#running-opctl-integration-tests)
+  - [Local Setup of AQUA API JupyterLab Server](#local-setup-of-aqua-api-jupyterlab-server)
+    - [Step 1: Requirements](#step-1-requirements)
+    - [Step 2: Create local .env files](#step-2-create-local-env-files)
+    - [Step 3: Add the run\_ads.sh script in the ADS Repository](#step-3-add-the-run_adssh-script-in-the-ads-repository)
+    - [Step 4: Run the JupyterLab Server](#step-4-run-the-jupyterlab-server)
+    - [Step 5: Run the unit tests for the AQUA API](#step-5-run-the-unit-tests-for-the-aqua-api)
+  - [Security](#security)
+  - [License](#license)
+
+
 ## Documentation
 
  - [ads-documentation](https://docs.oracle.com/en-us/iaas/tools/ads-sdk/latest/index.html)
@@ -137,6 +163,100 @@ To build development container, see the [Build Development Container Image](http
   python3 -m pytest tests/integration/opctl
 ```
 
+## Local Setup of AQUA API JupyterLab Server
+These are the steps to run the AQUA (AI Quick Actions) API Server for development and testing purposes. The source code for the AQUA API Server is [here](https://github.com/oracle/accelerated-data-science/tree/21ba00b95aef8581991fee6c7d558e2f2b1680ac/ads/aqua) within this repository.
+
+### Step 1: Requirements
++ Complete the [Getting Started](#getting-started) Section above, create a conda environment with python >3.9 or 3.10
++ install any Rest API Client in your IDE (Thunder Client on [vscode](https://marketplace.visualstudio.com/items?itemName=rangav.vscode-thunder-client) or Postman) 
++ Activate the conda environment from the Getting Started Section and run
+
+```
+pip install -r test-requirements.txt
+```
+
+### Step 2: Create local .env files 
+Running the local JuypterLab server requires setting OCI authentication, proxy, and OCI namespace parameters. Adapt this .env file with your specific OCI profile and OCIDs to set these variables.
+
+```
+CONDA_BUCKET_NS="your_conda_bucket"
+http_proxy=""
+https_proxy=""
+HTTP_PROXY=""
+HTTPS_PROXY=""
+OCI_ODSC_SERVICE_ENDPOINT="your_service_endpoint"
+AQUA_SERVICE_MODELS_BUCKET="service-managed-models"
+AQUA_TELEMETRY_BUCKET_NS="" 
+PROJECT_COMPARTMENT_OCID="ocid1.compartment.oc1.<your_ocid>" 
+OCI_CONFIG_PROFILE="your_oci_profile_name"
+OCI_IAM_TYPE="security_token" # no modification needed if using token-based auth
+TENANCY_OCID="ocid1.tenancy.oc1.<your_ocid>"
+AQUA_JOB_SUBNET_ID="ocid1.subnet.oc1.<your_ocid>"
+ODSC_MODEL_COMPARTMENT_OCID="ocid1.compartment.oc1.<your_ocid>" 
+PROJECT_OCID="ocid1.datascienceproject.oc1.<your_ocid>"
+```
+
+### Step 3: Add the run_ads.sh script in the ADS Repository 
++ add the shell script below and .env file from step 2 to your local directory of the cloned ADS Repository
++ Run ```chmox +x run_ads.sh``` after you create this script.
+```
+#!/bin/bash
+
+#### Check if a CLI command is provided
+if [ "$#" -lt 1 ]; then
+  echo "Usage: $0 <cli command>"
+  exit 1
+fi
+
+#### Load environment variables from .env file
+if [ -f .env ]; then
+  export $(grep -v '^#' .env.int | xargs)
+else
+  echo "Error: .env.int file not found!"
+  exit 1
+fi
+
+# Execute the CLI command
+"$@"
+```
+
+### Step 4: Run the JupyterLab Server 
+We can start the JupyterLab server using the following command
+
+``` 
+./run_ads.sh jupyter lab --no-browser --ServerApp.disable_check_xsrf=True
+```
++ run ```pkill jupyter-lab``` to kill the JupyterLab server and re-run server to reflect changes made locally to the AQUA API
++ to test if server is running via CLI, run this in terminal
+
+```
+./run_ads.sh ads aqua model list
+```
+
+To make calls to the API, use the link http://localhost:8888/aqua/insert_handler_here with a REST API Client like Thunder Client/ Postman.
+
+Examples of handlers
+```
+GET http://localhost:8888/aqua/model # calling the model_handler.py
+
+GET http://localhost:8888/aqua/deployments # calling the deployment_handler.py
+```
+Handlers can be found [here](https://github.com/oracle/accelerated-data-science/tree/21ba00b95aef8581991fee6c7d558e2f2b1680ac/ads/aqua/extension).
+
+### Step 5: Run the unit tests for the AQUA API
+All the unit tests can be found [here](https://github.com/oracle/accelerated-data-science/tree/main/tests/unitary/with_extras/aqua). 
+The following commands detail how the unit tests can be run.
+```
+# Run all tests in AQUA project
+python -m pytest -q tests/unitary/with_extras/aqua/test_deployment.py
+
+# Run all tests specific to a module within in AQUA project (ex. test_deployment.py, test_model.py, etc.)
+python -m pytest -q tests/unitary/with_extras/aqua/test_deployment.py
+
+# Run specific test method within the module (replace test_get_deployment_default_params with targeted test method)
+python -m pytest tests/unitary/with_extras/aqua/test_deployment.py -k "test_get_deployment_default_params"
+```
+
 ## Security
 
 Consult the [security guide](./SECURITY.md) for our responsible security

diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt
@@ -96,6 +96,12 @@ fsspec
 * Source code: https://github.com/intake/filesystem_spec
 * Project home: https://github.com/intake/filesystem_spec
 
+httpx
+* Copyright (c) 2021 ProjectDiscovery, Inc.
+* License: MIT License
+* Source code: https://github.com/projectdiscovery/httpx
+* Project home: https://github.com/projectdiscovery/httpx
+
 geopandas
 * Copyright  (c) 2013-2016, GeoPandas developers
 * License: BSD 3-Clause "New" or "Revised" License
@@ -391,6 +397,12 @@ tabulate
 * Source code: https://github.com/astanin/python-tabulate
 * Project home: https://github.com/astanin/python-tabulate
 
+tenacity
+* No listed copyright holder
+* License: Apache-2.0 license
+* Source code: https://github.com/jd/tenacity
+* Project home: https://github.com/jd/tenacity
+
 tensorflow
 * Copyright (c) 2023 Google, Inc.
 * License: Apache-2.0 license

diff --git a/ads/aqua/__init__.py b/ads/aqua/__init__.py
@@ -1,12 +1,13 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
 import os
+from logging import getLogger
 
 from ads import logger, set_auth
+from ads.aqua.client.client import AsyncClient, Client
 from ads.aqua.common.utils import fetch_service_compartment
 from ads.config import OCI_RESOURCE_PRINCIPAL_VERSION
 
@@ -19,6 +20,7 @@ def get_logger_level():
     return level
 
 
+logger = getLogger(__name__)
 logger.setLevel(get_logger_level())
 
 
@@ -27,7 +29,6 @@ def set_log_level(log_level: str):
 
     log_level = log_level.upper()
     logger.setLevel(log_level.upper())
-    logger.handlers[0].setLevel(log_level)
 
 
 if OCI_RESOURCE_PRINCIPAL_VERSION:

diff --git a/ads/aqua/app.py b/ads/aqua/app.py
@@ -1,9 +1,10 @@
 #!/usr/bin/env python
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import json
 import os
+import traceback
 from dataclasses import fields
 from typing import Dict, Union
 
@@ -23,7 +24,7 @@
 from ads.aqua.constants import UNKNOWN
 from ads.common import oci_client as oc
 from ads.common.auth import default_signer
-from ads.common.utils import extract_region
+from ads.common.utils import extract_region, is_path_exists
 from ads.config import (
     AQUA_TELEMETRY_BUCKET,
     AQUA_TELEMETRY_BUCKET_NS,
@@ -296,33 +297,46 @@ def get_config(self, model_id: str, config_file_name: str) -> Dict:
             raise AquaRuntimeError(f"Target model {oci_model.id} is not Aqua model.")
 
         config = {}
-        artifact_path = get_artifact_path(oci_model.custom_metadata_list)
+        # if the current model has a service model tag, then
+        if Tags.AQUA_SERVICE_MODEL_TAG in oci_model.freeform_tags:
+            base_model_ocid = oci_model.freeform_tags[Tags.AQUA_SERVICE_MODEL_TAG]
+            logger.info(
+                f"Base model found for the model: {oci_model.id}. "
+                f"Loading {config_file_name} for base model {base_model_ocid}."
+            )
+            base_model = self.ds_client.get_model(base_model_ocid).data
+            artifact_path = get_artifact_path(base_model.custom_metadata_list)
+        else:
+            logger.info(f"Loading {config_file_name} for model {oci_model.id}...")
+            artifact_path = get_artifact_path(oci_model.custom_metadata_list)
+
         if not artifact_path:
-            logger.error(
+            logger.debug(
                 f"Failed to get artifact path from custom metadata for the model: {model_id}"
             )
             return config
 
-        try:
-            config_path = f"{os.path.dirname(artifact_path)}/config/"
-            config = load_config(
-                config_path,
-                config_file_name=config_file_name,
-            )
-        except Exception:
-            # todo: temp fix for issue related to config load for byom models, update logic to choose the right path
+        config_path = f"{os.path.dirname(artifact_path)}/config/"
+        if not is_path_exists(config_path):
+            config_path = f"{artifact_path.rstrip('/')}/config/"
+
+        config_file_path = f"{config_path}{config_file_name}"
+        if is_path_exists(config_file_path):
             try:
-                config_path = f"{artifact_path.rstrip('/')}/config/"
                 config = load_config(
                     config_path,
                     config_file_name=config_file_name,
                 )
             except Exception:
-                pass
+                logger.debug(
+                    f"Error loading the {config_file_name} at path {config_path}.\n"
+                    f"{traceback.format_exc()}"
+                )
 
         if not config:
-            logger.error(
-                f"{config_file_name} is not available for the model: {model_id}. Check if the custom metadata has the artifact path set."
+            logger.debug(
+                f"{config_file_name} is not available for the model: {model_id}. "
+                f"Check if the custom metadata has the artifact path set."
             )
             return config
 

diff --git a/ads/aqua/client/__init__.py b/ads/aqua/client/__init__.py
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# Copyright (c) 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/