FEATURE MYJSL-369: Use compatabile secrets to install (#367)

* FEATURE MYJSL-369: Use compatabile secrets to install

* FEATURE MYJSL-369: Refactor code

* FEATURE MYJSL-369: Add tests to cover functionality

johnsnowlabs/py_models/jsl_secrets.py

@@ -2,7 +2,7 @@
 import json
 import os
 from pathlib import Path
-from typing import Optional, Union, Dict, List
+from typing import Dict, List, Optional, Union
 
 from pydantic import validator
 
@@ -12,13 +12,11 @@
 from johnsnowlabs.py_models.primitive import LibVersionIdentifier, Secret
 from johnsnowlabs.utils.enums import ProductName
 from johnsnowlabs.utils.file_utils import json_path_as_dict
-from johnsnowlabs.utils.my_jsl_api import (
-    get_user_licenses,
-    download_license,
-    get_access_token,
-    get_access_key_from_browser,
-    get_user_lib_secrets,
-)
+from johnsnowlabs.utils.my_jsl_api import (download_license,
+                                           get_access_key_from_browser,
+                                           get_access_token, get_secrets,
+                                           get_user_lib_secrets,
+                                           get_user_licenses)
 
 secret_json_keys = [
     "JSL_SECRET",
@@ -91,7 +89,7 @@ def hc_version_check(cls, HC_SECRET):
             ):
                 hc_validation_logged = True
                 print(
-                    f"🚨 Outdated Medical Secrets in license file. Version={HC_SECRET.split('-')[0]} but should be Version={settings.raw_version_medical}"
+                    f"🚨 Outdated Medical Secrets in license file. Version={(HC_SECRET.split('-')[0] if HC_SECRET else None)} but should be Version={settings.raw_version_medical}"
                 )
                 if settings.enforce_secret_on_version:
                     raise ValueError("Invalid HC Secret")
@@ -123,7 +121,7 @@ def ocr_version_check(cls, OCR_SECRET):
             ):
                 ocr_validation_logged = True
                 print(
-                    f"🚨 Outdated OCR Secrets in license file. Version={OCR_SECRET.split('-')[0]} but should be Version={settings.raw_version_ocr}"
+                    f"🚨 Outdated OCR Secrets in license file. Version={(OCR_SECRET.split('-')[0] if OCR_SECRET else None)} but should be Version={settings.raw_version_ocr}"
                 )
                 if settings.enforce_secret_on_version:
                     raise ValueError("Invalid OCR Secret")
@@ -266,6 +264,12 @@ def build_or_try_find_secrets(
             if not secrets and not force_browser:
                 # Search Env Vars
                 secrets = JslSecrets.search_env_vars()
+
+            if secrets and settings.enforce_versions and not JslSecrets.is_hc_secret_correct_version(secrets.HC_SECRET) and not JslSecrets.is_ocr_secret_correct_version(secrets.OCR_SECRET):
+                # Make sure secrets and versions re enforced
+                print("👷 Trying to install compatible secrets. Use nlp.settings.enforce_versions=False if you want to install outdated secrets.")
+                secrets = JslSecrets.enforce_versions(secrets)
+
         except Exception as err:
             print(
                 f"🚨 Failure Trying to read license {err}\n",
@@ -470,6 +474,34 @@ def from_json_file_path(secrets_path):
         f.close()
         return creds
 
+    @staticmethod
+    def enforce_versions(data: "JslSecrets") -> "JslSecrets":
+        secrets  = get_secrets(data.HC_LICENSE or data.OCR_LICENSE or data.JSL_FINANCE_LICENSE or data.JSL_LEGAL_LICENSE)
+        # Fix lib secrets in license data to correct version
+        ocr_candidates = list(
+            filter(
+                lambda x: x.version_secret == settings.raw_version_secret_ocr
+                and x.product == ProductName.ocr,
+                secrets,
+            )
+        )
+        hc_handidates = list(
+            filter(
+                lambda x: x.version_secret == settings.raw_version_secret_medical
+                and x.product == ProductName.hc,
+                secrets,
+            )
+        )
+        data.NLP_VERSION = settings.raw_version_nlp
+        if hc_handidates:
+            data.HC_SECRET = hc_handidates[0].secret
+            data.HC_VERSION = hc_handidates[0].version
+        if ocr_candidates:
+            data.OCR_SECRET = ocr_candidates[0].secret
+            data.OCR_VERSION = ocr_candidates[0].version
+
+        return data
+
     @staticmethod
     def from_access_token(access_token, license_number=0):
         licenses = get_user_licenses(access_token)

johnsnowlabs/utils/my_jsl_api.py

@@ -6,14 +6,15 @@
 import string
 import webbrowser
 from http.server import BaseHTTPRequestHandler, HTTPServer
-from typing import List, Dict
-
+from typing import Dict, List
 # imports related to get access token with PKCE Oauth
 from urllib import parse
 from urllib.request import Request, urlopen
 
 from johnsnowlabs.utils.enums import ProductName
 
+LICENSE_SERVER_ORIGIN = os.environ.get("LICENSE_SERVER_ORIGIN", "https://license.johnsnowlabs.com")
+
 MYJSL_ORIGIN = os.environ.get("MYJSL_ORIGIN", "https://my.johnsnowlabs.com")
 
 # save_path that license should be downloaded there
@@ -130,6 +131,18 @@ def get_access_token(email, password):
     access_token = data["data"]["getAccessToken"]["ok"]["token"]
     return access_token
 
+def get_secrets(license):
+    try:
+        data = http_request(url=f"{LICENSE_SERVER_ORIGIN}/johnsnowlabs/releases/", method="GET", access_token=license)
+        if data:
+            return [LibSecretResponse(
+                isLatest=r.get("is_latest"),
+                product=r.get("product"),
+                secret=r.get("secret"),
+                version=r.get("version"),
+            ) for r in data]
+    except Exception:
+        raise ValueError("Usage of invalid/expired license.")
 
 def get_user_lib_secrets(access_token):
     secrets_query = """query ReleasesQuery {
@@ -237,7 +250,7 @@ def get_user_license_choice(licenses):
 
 def open_authorized_url(url, in_colab=False):
     if in_colab:
-        from IPython.display import display, Javascript
+        from IPython.display import Javascript, display
 
         display(
             Javascript(
@@ -321,7 +334,7 @@ def do_GET(self):
         open_authorized_url(url, in_colab)
         httpd.handle_request()
         if in_colab:
-            from IPython.display import display, Javascript
+            from IPython.display import Javascript, display
 
             display(Javascript("document.body.removeChild(a);"))
 

tests/install.py

@@ -0,0 +1,56 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from johnsnowlabs import nlp, settings
+from johnsnowlabs.py_models.jsl_secrets import JslSecrets
+
+
+class EnforceVersions(unittest.TestCase):
+    def setUp(self) -> None:
+        nlp.settings.enforce_versions=True
+
+    @patch("johnsnowlabs.py_models.jsl_secrets.JslSecrets.enforce_versions")
+    def test_should_try_to_enforce_versions(self, mock_enforce_versions):
+        nlp.install(med_license=os.environ.get("TEST_VALID_LICENSE"), only_refresh_credentials=True, refresh_install=True)
+        self.assertTrue(mock_enforce_versions.called)
+
+    def test_should_install_compatible_secrets_with_valid_license(self):
+        nlp.install(med_license=os.environ.get("TEST_VALID_LICENSE"), only_refresh_credentials=True, refresh_install=True)
+        secrets = JslSecrets.from_jsl_home()
+        self.assertEqual(secrets.HC_LICENSE, os.environ.get("TEST_VALID_LICENSE"))
+        self.assertEqual(secrets.HC_VERSION, settings.raw_version_secret_medical)
+        self.assertEqual(secrets.OCR_VERSION, settings.raw_version_secret_ocr)
+
+    def test_should_fallback_to_existing_secrets_with_invalid_license(self):
+        nlp.install(med_license=os.environ.get("TEST_INVALID_LICENSE"), 
+                    ocr_secret="random-ocr-secret",
+                    enterprise_nlp_secret="random-medical-secret",
+                    only_refresh_credentials=True, refresh_install=True)
+        secrets = JslSecrets.from_jsl_home()
+        self.assertTrue(secrets.HC_LICENSE, os.environ.get("TEST_INVALID_LICENSE"))
+        self.assertEqual(secrets.HC_SECRET, "random-medical-secret")
+        self.assertEqual(secrets.OCR_SECRET, "random-ocr-secret")
+
+
+class DonotEnforceVersions(unittest.TestCase):
+    def setUp(self) -> None:
+        nlp.settings.enforce_versions=False
+        self.enterprise_nlp_secret = "nlp_random"
+        self.ocr_secret = "ocr_random"
+
+    def test_should_install_incompatible_secrets_from_input(self):
+        nlp.install(med_license=os.environ.get("TEST_VALID_LICENSE"),ocr_secret=self.ocr_secret, enterprise_nlp_secret=self.enterprise_nlp_secret, visual=True , only_refresh_credentials=True, refresh_install=True)
+        secrets = JslSecrets.from_jsl_home()
+        self.assertEqual(secrets.HC_LICENSE, os.environ.get("TEST_VALID_LICENSE"))
+        self.assertEqual(secrets.HC_SECRET, self.enterprise_nlp_secret)
+        self.assertEqual(secrets.OCR_SECRET, self.ocr_secret)
+
+    @patch("johnsnowlabs.py_models.jsl_secrets.JslSecrets.enforce_versions")
+    @patch("johnsnowlabs.py_models.jsl_secrets.JslSecrets.is_ocr_secret_correct_version")
+    @patch("johnsnowlabs.py_models.jsl_secrets.JslSecrets.is_hc_secret_correct_version")
+    def test_should_not_try_to_enforce_versions_if_input_secrets_are_compatible(self, mock_hc_secret_correct, mock_ocr_secret_correct, mock_enforce_versions):
+        nlp.install(med_license=os.environ.get("TEST_VALID_LICENSE"), only_refresh_credentials=True, refresh_install=True)
+        mock_hc_secret_correct.return_value = True
+        mock_ocr_secret_correct.return_value = True
+        self.assertFalse(mock_enforce_versions.called)

tests/jsl_secrets.py

@@ -0,0 +1,72 @@
+import unittest
+from unittest.mock import patch
+
+from johnsnowlabs import settings
+from johnsnowlabs.py_models.jsl_secrets import JslSecrets
+from johnsnowlabs.utils.my_jsl_api import LibSecretResponse
+
+
+class JslSecretsTestCase(unittest.TestCase):
+    def setUp(self):
+        self.license = "license"
+        self.valid_healthcare_secret =  settings.raw_version_secret_medical+ "-healthcare-secret"
+        self.valid_ocr_secret = settings.raw_version_secret_ocr +  "-ocr-secret"
+
+    @patch("johnsnowlabs.py_models.jsl_secrets.get_secrets")
+    def test_enforce_versions_should_retrieve_matching_secrets(self, mock_get_secrets):
+        mock_get_secrets.return_value = [
+            LibSecretResponse(
+                product="Healthcare NLP",
+                version=settings.raw_version_secret_medical,
+                secret=self.valid_healthcare_secret,
+                isLatest=True,
+            ),
+            LibSecretResponse(
+                product="Visual NLP",
+                version=settings.raw_version_secret_ocr,
+                secret=self.valid_ocr_secret,
+                isLatest=True,
+            ),
+            LibSecretResponse(
+                product="Healthcare NLP",
+                version="x.x",
+                secret="healthcare-invalid-secret",
+                isLatest=False,
+            ),
+            LibSecretResponse(
+                product="Visual NLP",
+                version="x.x",
+                secret="ocr-invalid-secret",
+                isLatest=False,
+            ),
+        ]
+        secrets = JslSecrets.enforce_versions(JslSecrets(OCR_LICENSE=self.license))
+        self.assertEqual(secrets.OCR_LICENSE, self.license)
+        self.assertEqual(secrets.HC_VERSION, settings.raw_version_secret_medical)
+        self.assertEqual(secrets.OCR_VERSION, settings.raw_version_secret_ocr)
+        self.assertEqual(secrets.HC_SECRET, self.valid_healthcare_secret)
+        self.assertEqual(secrets.OCR_SECRET, self.valid_ocr_secret)
+
+
+    @patch("johnsnowlabs.py_models.jsl_secrets.get_secrets")
+    def test_should_return_existing_secret_if_no_matching_secret_found(self, mock_get_secrets):
+        existing_secrets = JslSecrets(HC_LICENSE=self.license, OCR_SECRET="ocr-invalid-secret", HC_SECRET="healthcare-invalid-secret")
+        mock_get_secrets.return_value = [
+            LibSecretResponse(
+                product="Healthcare NLP",
+                version="x.x",
+                secret="healthcare-invalid-secret",
+                isLatest=False,
+            ),
+            LibSecretResponse(
+                product="Visual NLP",
+                version="x.x",
+                secret="ocr-invalid-secret",
+                isLatest=False,
+            ),
+        ]
+        secrets = JslSecrets.enforce_versions(existing_secrets)
+        self.assertEqual(secrets.HC_LICENSE, self.license)
+        self.assertEqual(secrets.HC_SECRET, existing_secrets.HC_SECRET)
+        self.assertEqual(secrets.OCR_SECRET,existing_secrets.OCR_SECRET)
+