diff --git a/.github/workflows/mkdocs-release.yml b/.github/workflows/mkdocs-release.yml index 8962b675e..1efb6e69c 100644 --- a/.github/workflows/mkdocs-release.yml +++ b/.github/workflows/mkdocs-release.yml @@ -2,26 +2,26 @@ name: mkdocs-release on: push: - branches: [ branch-* ] + branches: [branch-*] jobs: publish-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: "3.10" - name: Install ubuntu dependencies run: sudo apt-get install -y libxml2-dev libxslt-dev - name: install deps - run: pip3 install pip==22.0.3 && pip3 install 'git+https://github.com/logicalclocks/feature-store-api@master#egg=hsfs[docs]&subdirectory=python' mkdocs-minify-plugin + run: pip3 install 'git+https://github.com/logicalclocks/feature-store-api@master#egg=hsfs[docs]&subdirectory=python' - name: setup git run: | diff --git a/.github/workflows/mkdocs-test.yml b/.github/workflows/mkdocs-test.yml index d8644d237..6c6172ab1 100644 --- a/.github/workflows/mkdocs-test.yml +++ b/.github/workflows/mkdocs-test.yml @@ -8,19 +8,19 @@ jobs: steps: - name: Checkout main repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: "3.10" - name: Install ubuntu dependencies run: sudo apt-get install -y libxml2-dev libxslt-dev - name: install deps - run: pip3 install pip==22.0.3 && pip3 install 'git+https://github.com/logicalclocks/feature-store-api@master#egg=hsfs[docs]&subdirectory=python' mkdocs-minify-plugin + run: pip3 install 'git+https://github.com/logicalclocks/feature-store-api@master#egg=hsfs[docs]&subdirectory=python' linkchecker - name: setup git run: | diff --git a/docs/css/custom.css b/docs/css/custom.css index 2f406a8d7..e7900b811 100644 --- a/docs/css/custom.css +++ b/docs/css/custom.css @@ -1,5 +1,5 @@ -:root { - --md-primary-fg-color: #1EB382; +[data-md-color-scheme="hopsworks"] { + --md-primary-fg-color: #1eb382; --md-secondary-fg-color: #188a64; --md-tertiary-fg-color: #0d493550; --md-quaternary-fg-color: #fdfdfd; @@ -9,7 +9,6 @@ .md-footer__inner:not([hidden]) { display: none } - /* Lex did stuff here */ .svg_topnav{ width: 12px; @@ -24,6 +23,10 @@ box-shadow: 0 0 0 0; } +.md-tabs__item { + min-width: 2.5rem; +} + .md-tabs__item:hover { background-color: var(--md-tertiary-fg-color); transition: background-color 450ms; diff --git a/docs/css/dropdown.css b/docs/css/dropdown.css index e1d672432..c1c768fa2 100644 --- a/docs/css/dropdown.css +++ b/docs/css/dropdown.css @@ -17,7 +17,7 @@ overflow: inherit; /* The container
- needed to position the dropdown content */ .dropdown { - position: relative; + position: absolute; display: inline-block; } diff --git a/docs/js/inject-api-links.js b/docs/js/inject-api-links.js index 8e05a50b6..156650f66 100644 --- a/docs/js/inject-api-links.js +++ b/docs/js/inject-api-links.js @@ -1,19 +1,25 @@ window.addEventListener("DOMContentLoaded", function () { var windowPathNameSplits = window.location.pathname.split("/"); var majorVersionRegex = new RegExp("(\\d+[.]\\d+)") + var latestRegex = new RegExp("latest") if (majorVersionRegex.test(windowPathNameSplits[1])) { // On landing page docs.hopsworks.api/3.0 - URL contains major version // Version API dropdown document.getElementById("hopsworks_api_link").href = "https://docs.hopsworks.ai/hopsworks-api/" + windowPathNameSplits[1] + "/generated/api/login/"; document.getElementById("hsfs_api_link").href = "https://docs.hopsworks.ai/feature-store-api/" + windowPathNameSplits[1] + "/generated/api/connection_api/"; - document.getElementById("hsml_api_link").href = "https://docs.hopsworks.ai/machine-learning-api/" + windowPathNameSplits[1] + "/generated/connection_api/"; document.getElementById("hsfs_javadoc_link").href = "https://docs.hopsworks.ai/feature-store-api/" + windowPathNameSplits[1] + "/javadoc"; + document.getElementById("hsml_api_link").href = "https://docs.hopsworks.ai/machine-learning-api/" + windowPathNameSplits[1] + "/generated/connection_api/"; } else { // on docs.hopsworks.api/feature-store-api/3.0 / docs.hopsworks.api/hopsworks-api/3.0 / docs.hopsworks.api/machine-learning-api/3.0 - var apiVersion = windowPathNameSplits[2]; - var majorVersion = apiVersion.match(majorVersionRegex)[0]; + + if (latestRegex.test(windowPathNameSplits[2]) || latestRegex.test(windowPathNameSplits[1])) { + var majorVersion = "latest"; + } else { + var apiVersion = windowPathNameSplits[2]; + var majorVersion = apiVersion.match(majorVersionRegex)[0]; + } // Version main navigation document.getElementsByClassName("md-tabs__link")[0].href = "https://docs.hopsworks.ai/" + majorVersion; - document.getElementsByClassName("md-tabs__link")[1].href = "https://docs.hopsworks.ai/" + majorVersion + "/getting_started/quickstart/"; - document.getElementsByClassName("md-tabs__link")[2].href = "https://docs.hopsworks.ai/" + majorVersion + "/tutorials/fraud_batch/1_feature_groups/"; + document.getElementsByClassName("md-tabs__link")[1].href = "https://colab.research.google.com/github/logicalclocks/hopsworks-tutorials/blob/master/quickstart.ipynb"; + document.getElementsByClassName("md-tabs__link")[2].href = "https://docs.hopsworks.ai/" + majorVersion + "/tutorials/"; document.getElementsByClassName("md-tabs__link")[3].href = "https://docs.hopsworks.ai/" + majorVersion + "/concepts/hopsworks/"; document.getElementsByClassName("md-tabs__link")[4].href = "https://docs.hopsworks.ai/" + majorVersion + "/user_guides/"; document.getElementsByClassName("md-tabs__link")[5].href = "https://docs.hopsworks.ai/" + majorVersion + "/setup_installation/aws/getting_started/"; diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 000000000..7fc85fb25 --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} + +{% block outdated %} +You're not viewing the latest version of the documentation. + + Click here to go to latest. + +{% endblock %} \ No newline at end of file diff --git a/docs/user_guides/fs/feature_group/create_external.md b/docs/user_guides/fs/feature_group/create_external.md index a8a7881fe..e349b9006 100644 --- a/docs/user_guides/fs/feature_group/create_external.md +++ b/docs/user_guides/fs/feature_group/create_external.md @@ -93,13 +93,39 @@ In the snippet above it's important that the created metadata object gets regist fg.save() ``` -### Limitations +### Enable online storage -Hopsworks Feature Store does not support time-travel capabilities for external feature groups. Moreover, as the data resides on external systems, external feature groups cannot be made available online for low latency serving. To make data from an external feature group available online, users need to define an online enabled feature group and have a job that periodically reads data from the external feature group and writes in the online feature group. +You can enable online storage for external feature groups, however, the sync from the external storage to Hopsworks online storage is not automatic and needs to be setup manually. For an external feature group to be available online, during the creation of the feature group, the `online_enabled` option needs to be set to `True`. -!!! warning "Python support" +=== "Python" + + ```python + external_fg = fs.create_external_feature_group( + name="sales", + version=1, + description="Physical shop sales features", + query=query, + storage_connector=connector, + primary_key=['ss_store_sk'], + event_time='sale_date', + online_enabled=True) + external_fg.save() + + # read from external storage and filter data to sync to online + df = external_fg.read().filter(external_fg.customer_status == "active") + + # insert to online storage + external_fg.insert(df) + ``` + +The `insert()` method takes a DataFrame as parameter and writes it _only_ to the online feature store. Users can select which subset of the feature group data they want to make available on the online feautre store by using the [query APIs](https://docs.hopsworks.ai/feature-store-api/{{{ hopsworks_version }}}/generated/api/query_api/). + +### Limitations + +Hopsworks Feature Store does not support time-travel queries on external feature groups. - Currently the HSFS library does not support calling the read() or show() methods on external feature groups. Likewise it is not possible to call the read() or show() methods on queries containing external feature groups. Nevertheless, external feature groups can be used from a Python engine to create training datasets. +Additionally, support for `.read()` and `.show()` methods when using by the Python engine is limited to external feature groups defined on BigQuery and Snowflake and only when using the [Feature Query Service](../../../setup_installation/common/arrow_flight_duckdb.md). +Nevertheless, external feature groups defined top of any storage connector can be used to create a training dataset from a Python environment invoking one of the following methods: [create_training_data](https://docs.hopsworks.ai/feature-store-api/{{{ hopsworks_version }}}/generated/api/feature_view_api/#create_training_data), [create_train_test_split](https://docs.hopsworks.ai/feature-store-api/{{{ hopsworks_version }}}/generated/api/feature_view_api/#create_train_test_split) or the [create_train_validation_test_split](https://docs.hopsworks.ai/feature-store-api/{{{ hopsworks_version }}}/generated/api/feature_view_api/#create_train_validation_test_split) ### API Reference diff --git a/mkdocs.yml b/mkdocs.yml index 395ce9ef9..88640e72a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -261,6 +261,7 @@ nav: theme: name: material + custom_dir: docs/overrides favicon: assets/images/favicon.ico logo: assets/images/hops-logo.png icon: @@ -270,6 +271,7 @@ theme: code: "IBM Plex Mono" palette: accent: teal + scheme: hopsworks features: - navigation.tabs - navigation.tabs.sticky @@ -280,6 +282,7 @@ extra: hopsworks_version: 3.5 version: provider: mike + default: latest generator: false social: - icon: fontawesome/brands/twitter @@ -344,5 +347,5 @@ markdown_extensions: - markdown_include.include: base_path: docs - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg