[FSTORE-1346][FSTORE-1212] Upgrade dependencies, fix theme, latest re…

…direct, latest version banner (#371) * [HWORKS-742] Update documentation on external feature groups limitations (#363) * Add overrides for banner showing not viewing latest version (#360) * [FSTORE-1346] Fixing workflow for docs release (#364) (#365) * Fixing workflow for docs release * Fixing workflow for doc release test also * Minimal changes * Add support for latest redirect --------- Co-authored-by: Fabio Buso <[email protected]>
logicalclocks · Apr 11, 2024 · 2a5af0a · 2a5af0a
1 parent 132c14b
commit 2a5af0a
Show file tree

Hide file tree

Showing 8 changed files with 70 additions and 24 deletions.
diff --git a/.github/workflows/mkdocs-release.yml b/.github/workflows/mkdocs-release.yml
@@ -2,26 +2,26 @@ name: mkdocs-release
 
 on:
   push:
-    branches: [ branch-* ]
+    branches: [branch-*]
 
 jobs:
   publish-release:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: "3.10"
 
       - name: Install ubuntu dependencies
         run: sudo apt-get install -y libxml2-dev libxslt-dev
 
       - name: install deps
-        run: pip3 install pip==22.0.3 && pip3 install 'git+https://github.com/logicalclocks/feature-store-api@master#egg=hsfs[docs]&subdirectory=python' mkdocs-minify-plugin
+        run: pip3 install 'git+https://github.com/logicalclocks/feature-store-api@master#egg=hsfs[docs]&subdirectory=python'
 
       - name: setup git
         run: |

diff --git a/.github/workflows/mkdocs-test.yml b/.github/workflows/mkdocs-test.yml
@@ -8,19 +8,19 @@ jobs:
 
     steps:
       - name: Checkout main repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: "3.10"
 
       - name: Install ubuntu dependencies
         run: sudo apt-get install -y libxml2-dev libxslt-dev
 
       - name: install deps
-        run: pip3 install pip==22.0.3 && pip3 install 'git+https://github.com/logicalclocks/feature-store-api@master#egg=hsfs[docs]&subdirectory=python' mkdocs-minify-plugin
+        run: pip3 install 'git+https://github.com/logicalclocks/feature-store-api@master#egg=hsfs[docs]&subdirectory=python' linkchecker
 
       - name: setup git
         run: |

diff --git a/docs/css/custom.css b/docs/css/custom.css
@@ -1,5 +1,5 @@
-:root {
-  --md-primary-fg-color: #1EB382;
+[data-md-color-scheme="hopsworks"] {
+  --md-primary-fg-color: #1eb382;
   --md-secondary-fg-color: #188a64;
   --md-tertiary-fg-color: #0d493550;
   --md-quaternary-fg-color: #fdfdfd;
@@ -9,7 +9,6 @@
 .md-footer__inner:not([hidden]) {
   display: none
 }
-
 /* Lex did stuff here */
 .svg_topnav{
   width: 12px;
@@ -24,6 +23,10 @@
   box-shadow: 0 0 0 0;
 }
 
+.md-tabs__item {
+  min-width: 2.5rem;
+}
+
 .md-tabs__item:hover {
   background-color: var(--md-tertiary-fg-color);
   transition: background-color 450ms;

diff --git a/docs/css/dropdown.css b/docs/css/dropdown.css
@@ -17,7 +17,7 @@ overflow: inherit;
 
 /* The container <div> - needed to position the dropdown content */
 .dropdown {
-  position: relative;
+  position: absolute;
   display: inline-block;
 }
 

diff --git a/docs/js/inject-api-links.js b/docs/js/inject-api-links.js
@@ -1,19 +1,25 @@
 window.addEventListener("DOMContentLoaded", function () {
     var windowPathNameSplits = window.location.pathname.split("/");
     var majorVersionRegex = new RegExp("(\\d+[.]\\d+)")
+    var latestRegex = new RegExp("latest")
     if (majorVersionRegex.test(windowPathNameSplits[1])) { // On landing page docs.hopsworks.api/3.0 - URL contains major version
         // Version API dropdown
         document.getElementById("hopsworks_api_link").href = "https://docs.hopsworks.ai/hopsworks-api/" + windowPathNameSplits[1] + "/generated/api/login/";
         document.getElementById("hsfs_api_link").href = "https://docs.hopsworks.ai/feature-store-api/" + windowPathNameSplits[1] + "/generated/api/connection_api/";
-        document.getElementById("hsml_api_link").href = "https://docs.hopsworks.ai/machine-learning-api/" + windowPathNameSplits[1] + "/generated/connection_api/";
         document.getElementById("hsfs_javadoc_link").href = "https://docs.hopsworks.ai/feature-store-api/" + windowPathNameSplits[1] + "/javadoc";
+        document.getElementById("hsml_api_link").href = "https://docs.hopsworks.ai/machine-learning-api/" + windowPathNameSplits[1] + "/generated/connection_api/";
     } else { // on docs.hopsworks.api/feature-store-api/3.0 / docs.hopsworks.api/hopsworks-api/3.0 / docs.hopsworks.api/machine-learning-api/3.0
-        var apiVersion = windowPathNameSplits[2];
-        var majorVersion = apiVersion.match(majorVersionRegex)[0];
+
+        if (latestRegex.test(windowPathNameSplits[2]) || latestRegex.test(windowPathNameSplits[1])) {
+            var majorVersion = "latest";
+        } else {
+            var apiVersion = windowPathNameSplits[2];
+            var majorVersion = apiVersion.match(majorVersionRegex)[0];
+        }
         // Version main navigation
         document.getElementsByClassName("md-tabs__link")[0].href = "https://docs.hopsworks.ai/" + majorVersion;
-        document.getElementsByClassName("md-tabs__link")[1].href = "https://docs.hopsworks.ai/" + majorVersion + "/getting_started/quickstart/";
-        document.getElementsByClassName("md-tabs__link")[2].href = "https://docs.hopsworks.ai/" + majorVersion + "/tutorials/fraud_batch/1_feature_groups/";
+        document.getElementsByClassName("md-tabs__link")[1].href = "https://colab.research.google.com/github/logicalclocks/hopsworks-tutorials/blob/master/quickstart.ipynb";
+        document.getElementsByClassName("md-tabs__link")[2].href = "https://docs.hopsworks.ai/" + majorVersion + "/tutorials/";
         document.getElementsByClassName("md-tabs__link")[3].href = "https://docs.hopsworks.ai/" + majorVersion + "/concepts/hopsworks/";
         document.getElementsByClassName("md-tabs__link")[4].href = "https://docs.hopsworks.ai/" + majorVersion + "/user_guides/";
         document.getElementsByClassName("md-tabs__link")[5].href = "https://docs.hopsworks.ai/" + majorVersion + "/setup_installation/aws/getting_started/";

diff --git a/docs/overrides/main.html b/docs/overrides/main.html
@@ -0,0 +1,8 @@
+{% extends "base.html" %}
+
+{% block outdated %}
+You're not viewing the latest version of the documentation.
+<a href="{{ '../' ~ base_url }}">
+    <strong>Click here to go to latest.</strong>
+</a>
+{% endblock %}
diff --git a/docs/user_guides/fs/feature_group/create_external.md b/docs/user_guides/fs/feature_group/create_external.md
@@ -93,13 +93,39 @@ In the snippet above it's important that the created metadata object gets regist
     fg.save()
     ```
 
-### Limitations 
+### Enable online storage
 
-Hopsworks Feature Store does not support time-travel capabilities for external feature groups. Moreover, as the data resides on external systems, external feature groups cannot be made available online for low latency serving. To make data from an external feature group available online, users need to define an online enabled feature group and have a job that periodically reads data from the external feature group and writes in the online feature group.
+You can enable online storage for external feature groups, however, the sync from the external storage to Hopsworks online storage is not automatic and needs to be setup manually. For an external feature group to be available online, during the creation of the feature group, the `online_enabled` option needs to be set to `True`.
 
-!!! warning "Python support"
+=== "Python"
+
+    ```python
+    external_fg = fs.create_external_feature_group(
+                name="sales",
+                version=1,
+                description="Physical shop sales features",
+                query=query,
+                storage_connector=connector,
+                primary_key=['ss_store_sk'],
+                event_time='sale_date',
+                online_enabled=True)
+    external_fg.save()
+
+    # read from external storage and filter data to sync to online
+    df = external_fg.read().filter(external_fg.customer_status == "active")
+
+    # insert to online storage
+    external_fg.insert(df)
+    ```
+
+The `insert()` method takes a DataFrame as parameter and writes it _only_ to the online feature store. Users can select which subset of the feature group data they want to make available on the online feautre store by using the [query APIs](https://docs.hopsworks.ai/feature-store-api/{{{ hopsworks_version }}}/generated/api/query_api/).
+
+### Limitations
+
+Hopsworks Feature Store does not support time-travel queries on external feature groups.
 
-    Currently the HSFS library does not support calling the read() or show() methods on external feature groups. Likewise it is not possible to call the read() or show() methods on queries containing external feature groups. Nevertheless, external feature groups can be used from a Python engine to create training datasets.
+Additionally, support for `.read()` and `.show()` methods when using by the Python engine is limited to external feature groups defined on BigQuery and Snowflake and only when using the [Feature Query Service](../../../setup_installation/common/arrow_flight_duckdb.md).
+Nevertheless, external feature groups defined top of any storage connector can be used to create a training dataset from a Python environment invoking one of the following methods: [create_training_data](https://docs.hopsworks.ai/feature-store-api/{{{ hopsworks_version }}}/generated/api/feature_view_api/#create_training_data), [create_train_test_split](https://docs.hopsworks.ai/feature-store-api/{{{ hopsworks_version }}}/generated/api/feature_view_api/#create_train_test_split) or the [create_train_validation_test_split](https://docs.hopsworks.ai/feature-store-api/{{{ hopsworks_version }}}/generated/api/feature_view_api/#create_train_validation_test_split)
 
 
 ### API Reference 

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -261,6 +261,7 @@ nav:
 
 theme:
   name: material
+  custom_dir: docs/overrides
   favicon: assets/images/favicon.ico
   logo: assets/images/hops-logo.png
   icon:
@@ -270,6 +271,7 @@ theme:
     code: "IBM Plex Mono"
   palette:
     accent: teal
+    scheme: hopsworks
   features:
     - navigation.tabs
     - navigation.tabs.sticky
@@ -280,6 +282,7 @@ extra:
   hopsworks_version: 3.5
   version:
     provider: mike
+    default: latest
   generator: false
   social:
     - icon: fontawesome/brands/twitter
@@ -344,5 +347,5 @@ markdown_extensions:
   - markdown_include.include:
       base_path: docs
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg