[FSTORE-1439] Merge hopsworks-api, feature-store-api and machine-lear…

…ning-api

* Remove redundant java-related files

* Move java code of hsfs

* Move locust_benchmark of hsfs

* Move python code of hsfs

* Move python code of hsml

* Move hsml tests

* Remove redundant __init__.py in hsml/tests

* Move hsfs tests

* Merge hsfs tests

* Fix the problem with impure tests changing backend_fixtures

One of the tests is impure, changing backend_fixtures, which breaks test_model.
This commit makes backend_fixtures to return a deep copy of backend_fixtures_json isntead of returning them directly.

* Fix ruff check of test_base_client

* Move hsfs utils

* Merge hsfs docs

* Fix pyproject dependencies for hsfs docs

* Merge the rest of hsfs docs

* Merge hsfs mkdocs

* Merge hsml docs

* Add files generated by auto_doc to gitignore

* Merge hsfs pyproject.toml

* Merge hsml pyproject.toml

* Remove hsfs and hsml pyproject.toml

* Remove hive from pyproject dependencies

* Fix mistypes in pyproject

* Fix documentation generation

* Fix circular dependencies

* Fix circular dependency in hsml (CONNECTION_SAAS_HOSTNAME)

* Fix circular dependency in hsml (model/util)

* Ruff fix hsml/util

* Fix docgen

There is a very strange bug: hsml.python for some reason is available only as python.hsml.python.

* Fix hsml.python import errors

* Fix import problem in docgen

* Ruff fix auto_doc

* Revert the move of requirements-docs to an extra

See PR #209

* Move workflows

* Merge workflows

* Remove redundant github files

* Skip test_login until Robin is back

* Fix test_connection

* Fix test_hopsworks_udf

* Fix import problem in docgen

* Fix pyproject optional docgen dependency

* Revert the move of requirements-docs to an extra

See PR #209

* Rename unit_tests_pandas to unit_tests_pandas1 for clarity

* Rename unit_tests_typechecked

hsfs/.github/workflows/java-ut.yml → .github/workflows/java.yml

@@ -3,8 +3,8 @@ name: java
 on: pull_request
 
 jobs:
-  unit_tests_utc:
-    name: Java Unit Tests
+  unit_tests:
+    name: Unit Tests
     runs-on: ubuntu-latest
 
     steps:
@@ -17,8 +17,8 @@ jobs:
       - name: Set up JDK 8
         uses: actions/setup-java@v3
         with:
-          java-version: '8'
-          distribution: 'adopt'
+          java-version: "8"
+          distribution: "adopt"
 
       - name: Cache local Maven repository
         uses: actions/cache@v2
@@ -32,8 +32,8 @@ jobs:
         working-directory: ./java
         run: mvn clean test
 
-  unit_tests_local:
-    name: Java Unit Tests (Local TZ)
+  unit_tests_local_tz:
+    name: Unit Tests (Local TZ)
     runs-on: ubuntu-latest
 
     steps:
@@ -46,8 +46,8 @@ jobs:
       - name: Set up JDK 8
         uses: actions/setup-java@v3
         with:
-          java-version: '8'
-          distribution: 'adopt'
+          java-version: "8"
+          distribution: "adopt"
 
       - name: Cache local Maven repository
         uses: actions/cache@v2

.github/workflows/mkdocs-main.yml

@@ -24,7 +24,24 @@ jobs:
         run: cp ../README.md . && pip3 install -r ../requirements-docs.txt && pip3 install -e .[dev]
 
       - name: generate autodoc
-        run: python3 auto_doc.py
+        run: python3 ./python/auto_doc.py
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "adopt"
+
+      - name: Build java doc documentation
+        working-directory: ./java
+        run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc
 
       - name: setup git
         run: |

.github/workflows/mkdocs-release.yml

@@ -2,7 +2,7 @@ name: mkdocs-release
 
 on:
   push:
-    branches: [branch-*\.*]
+    branches: [branch-*]
 
 jobs:
   publish-release:
@@ -29,7 +29,25 @@ jobs:
         run: cp ../README.md . && pip3 install -r ../requirements-docs.txt && pip3 install -e .[dev]
 
       - name: generate autodoc
-        run: python3 auto_doc.py
+        run: python3 ./python/auto_doc.py
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "adopt"
+
+      - name: Build java doc documentation
+        working-directory: ./java
+        run:
+          mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc
 
       - name: setup git
         run: |

hsfs/.github/workflows/python-lint.yml → .github/workflows/python.yml

@@ -2,6 +2,10 @@ name: python
 
 on: pull_request
 
+env:
+  APP_API_KEY: ${{ secrets.APP_API_KEY }}
+  ENABLE_HOPSWORKS_USAGE: "false"
+
 jobs:
   lint_stylecheck:
     name: Lint and Stylecheck
@@ -31,22 +35,25 @@ jobs:
       - name: ruff on python files
         if: steps.get-changed-files.outputs.src_any_changed == 'true'
         env:
-          SRC_ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.src_all_changed_files }}
+          SRC_ALL_CHANGED_FILES:
+            ${{ steps.get-changed-files.outputs.src_all_changed_files }}
         run: ruff check --output-format=github $SRC_ALL_CHANGED_FILES
 
       - name: ruff on test files
         if: steps.get-changed-files.outputs.test_any_changed == 'true'
         env:
-          TEST_ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.test_all_changed_files }}
+          TEST_ALL_CHANGED_FILES:
+            ${{ steps.get-changed-files.outputs.test_all_changed_files }}
         run: ruff check --output-format=github $TEST_ALL_CHANGED_FILES
 
       - name: ruff format --check $ALL_CHANGED_FILES
         env:
-          ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.all_changed_files }}
+          ALL_CHANGED_FILES:
+            ${{ steps.get-changed-files.outputs.all_changed_files }}
         run: ruff format $ALL_CHANGED_FILES
 
-  unit_tests_ubuntu_utc:
-    name: Unit Testing (Ubuntu)
+  unit_tests:
+    name: Unit Tests
     needs: lint_stylecheck
     runs-on: ubuntu-latest
     strategy:
@@ -73,12 +80,34 @@ jobs:
         run: python --version
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests
 
-  unit_tests_ubuntu_pandas:
-    name: Unit Testing (Ubuntu) (Pandas 1.x)
+  unit_tests_no_opt:
+    name: Unit Tests (No Optional Dependencies)
+    needs: lint_stylecheck
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set Timezone
+        run: sudo timedatectl set-timezone UTC
+
+      - uses: actions/checkout@v4
+      - name: Copy README
+        run: cp README.md python/
+
+      - uses: actions/setup-python@v5
+        name: Setup Python
+        with:
+          python-version: "3.10"
+          cache: "pip"
+          cache-dependency-path: "python/setup.py"
+      - run: pip install -e python[python,dev-no-opt]
+
+      - name: Run Pytest suite
+        run: pytest python/tests
+
+  unit_tests_pandas1:
+    name: Unit Tests (Pandas 1.x)
     needs: lint_stylecheck
     runs-on: ubuntu-latest
 
@@ -102,12 +131,10 @@ jobs:
         run: python --version
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests
 
-  unit_tests_ubuntu_local:
-    name: Unit Testing (Ubuntu) (Local TZ)
+  unit_tests_local_tz:
+    name: Unit Tests (Local TZ)
     needs: lint_stylecheck
     runs-on: ubuntu-latest
 
@@ -131,12 +158,10 @@ jobs:
         run: python --version
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests
 
-  unit_tests_ubuntu_typechecked:
-    name: Typechecked Unit Testing (Ubuntu)
+  unit_tests_typechecked:
+    name: Unit Tests (Typechecked)
     needs: lint_stylecheck
     runs-on: ubuntu-latest
 
@@ -155,13 +180,12 @@ jobs:
 
       - name: Run Pytest suite
         env:
-          ENABLE_HOPSWORKS_USAGE: "false"
           HOPSWORKS_RUN_WITH_TYPECHECK: "true"
         run: pytest python/tests
         continue-on-error: true
 
-  unit_tests_windows_utc:
-    name: Unit Testing (Windows)
+  unit_tests_windows:
+    name: Unit Tests (Windows)
     needs: lint_stylecheck
     runs-on: windows-latest
 
@@ -185,12 +209,10 @@ jobs:
         run: python --version
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests
 
-  unit_tests_windows_local:
-    name: Unit Testing (Windows) (Local TZ)
+  unit_tests_windows_local_tz:
+    name: Unit Tests (Windows) (Local TZ)
     needs: lint_stylecheck
     runs-on: windows-latest
 
@@ -217,6 +239,4 @@ jobs:
         run: pip freeze
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests

.gitignore

@@ -128,3 +128,6 @@ target/
 
 # mkdocs intemediate files
 docs/generated
+
+docs/CONTRIBUTING.md
+docs/index.md