[FSTORE-1464] Unify tags_api, inode and dataset_api

hsfs/.github/workflows/java-ut.yml → .github/workflows/java.yml

@@ -3,8 +3,8 @@ name: java
 on: pull_request
 
 jobs:
-  unit_tests_utc:
-    name: Java Unit Tests
+  unit_tests:
+    name: Unit Tests
     runs-on: ubuntu-latest
 
     steps:
@@ -17,8 +17,8 @@ jobs:
       - name: Set up JDK 8
         uses: actions/setup-java@v3
         with:
-          java-version: '8'
-          distribution: 'adopt'
+          java-version: "8"
+          distribution: "adopt"
 
       - name: Cache local Maven repository
         uses: actions/cache@v2
@@ -32,8 +32,8 @@ jobs:
         working-directory: ./java
         run: mvn clean test
 
-  unit_tests_local:
-    name: Java Unit Tests (Local TZ)
+  unit_tests_local_tz:
+    name: Unit Tests (Local TZ)
     runs-on: ubuntu-latest
 
     steps:
@@ -46,8 +46,8 @@ jobs:
       - name: Set up JDK 8
         uses: actions/setup-java@v3
         with:
-          java-version: '8'
-          distribution: 'adopt'
+          java-version: "8"
+          distribution: "adopt"
 
       - name: Cache local Maven repository
         uses: actions/cache@v2

.github/workflows/mkdocs-main.yml

@@ -24,7 +24,24 @@ jobs:
         run: cp ../README.md . && pip3 install -r ../requirements-docs.txt && pip3 install -e .[dev]
 
       - name: generate autodoc
-        run: python3 auto_doc.py
+        run: python3 ./python/auto_doc.py
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "adopt"
+
+      - name: Build java doc documentation
+        working-directory: ./java
+        run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc
 
       - name: setup git
         run: |

.github/workflows/mkdocs-release.yml

@@ -2,7 +2,7 @@ name: mkdocs-release
 
 on:
   push:
-    branches: [branch-*\.*]
+    branches: [branch-*]
 
 jobs:
   publish-release:
@@ -29,7 +29,25 @@ jobs:
         run: cp ../README.md . && pip3 install -r ../requirements-docs.txt && pip3 install -e .[dev]
 
       - name: generate autodoc
-        run: python3 auto_doc.py
+        run: python3 ./python/auto_doc.py
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "adopt"
+
+      - name: Build java doc documentation
+        working-directory: ./java
+        run:
+          mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc
 
       - name: setup git
         run: |

hsfs/.github/workflows/python-lint.yml → .github/workflows/python.yml

@@ -2,6 +2,10 @@ name: python
 
 on: pull_request
 
+env:
+  APP_API_KEY: ${{ secrets.APP_API_KEY }}
+  ENABLE_HOPSWORKS_USAGE: "false"
+
 jobs:
   lint_stylecheck:
     name: Lint and Stylecheck
@@ -31,22 +35,25 @@ jobs:
       - name: ruff on python files
         if: steps.get-changed-files.outputs.src_any_changed == 'true'
         env:
-          SRC_ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.src_all_changed_files }}
+          SRC_ALL_CHANGED_FILES:
+            ${{ steps.get-changed-files.outputs.src_all_changed_files }}
         run: ruff check --output-format=github $SRC_ALL_CHANGED_FILES
 
       - name: ruff on test files
         if: steps.get-changed-files.outputs.test_any_changed == 'true'
         env:
-          TEST_ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.test_all_changed_files }}
+          TEST_ALL_CHANGED_FILES:
+            ${{ steps.get-changed-files.outputs.test_all_changed_files }}
         run: ruff check --output-format=github $TEST_ALL_CHANGED_FILES
 
       - name: ruff format --check $ALL_CHANGED_FILES
         env:
-          ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.all_changed_files }}
+          ALL_CHANGED_FILES:
+            ${{ steps.get-changed-files.outputs.all_changed_files }}
         run: ruff format $ALL_CHANGED_FILES
 
-  unit_tests_ubuntu_utc:
-    name: Unit Testing (Ubuntu)
+  unit_tests:
+    name: Unit Tests
     needs: lint_stylecheck
     runs-on: ubuntu-latest
     strategy:
@@ -73,12 +80,34 @@ jobs:
         run: python --version
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests
 
-  unit_tests_ubuntu_pandas:
-    name: Unit Testing (Ubuntu) (Pandas 1.x)
+  unit_tests_no_opt:
+    name: Unit Tests (No Optional Dependencies)
+    needs: lint_stylecheck
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set Timezone
+        run: sudo timedatectl set-timezone UTC
+
+      - uses: actions/checkout@v4
+      - name: Copy README
+        run: cp README.md python/
+
+      - uses: actions/setup-python@v5
+        name: Setup Python
+        with:
+          python-version: "3.10"
+          cache: "pip"
+          cache-dependency-path: "python/setup.py"
+      - run: pip install -e python[python,dev-no-opt]
+
+      - name: Run Pytest suite
+        run: pytest python/tests
+
+  unit_tests_pandas1:
+    name: Unit Tests (Pandas 1.x)
     needs: lint_stylecheck
     runs-on: ubuntu-latest
 
@@ -102,12 +131,10 @@ jobs:
         run: python --version
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests
 
-  unit_tests_ubuntu_local:
-    name: Unit Testing (Ubuntu) (Local TZ)
+  unit_tests_local_tz:
+    name: Unit Tests (Local TZ)
     needs: lint_stylecheck
     runs-on: ubuntu-latest
 
@@ -131,12 +158,10 @@ jobs:
         run: python --version
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests
 
-  unit_tests_ubuntu_typechecked:
-    name: Typechecked Unit Testing (Ubuntu)
+  unit_tests_typechecked:
+    name: Unit Tests (Typechecked)
     needs: lint_stylecheck
     runs-on: ubuntu-latest
 
@@ -155,13 +180,12 @@ jobs:
 
       - name: Run Pytest suite
         env:
-          ENABLE_HOPSWORKS_USAGE: "false"
           HOPSWORKS_RUN_WITH_TYPECHECK: "true"
         run: pytest python/tests
         continue-on-error: true
 
-  unit_tests_windows_utc:
-    name: Unit Testing (Windows)
+  unit_tests_windows:
+    name: Unit Tests (Windows)
     needs: lint_stylecheck
     runs-on: windows-latest
 
@@ -185,12 +209,10 @@ jobs:
         run: python --version
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests
 
-  unit_tests_windows_local:
-    name: Unit Testing (Windows) (Local TZ)
+  unit_tests_windows_local_tz:
+    name: Unit Tests (Windows) (Local TZ)
     needs: lint_stylecheck
     runs-on: windows-latest
 
@@ -217,6 +239,4 @@ jobs:
         run: pip freeze
 
       - name: Run Pytest suite
-        env:
-          ENABLE_HOPSWORKS_USAGE: "false"
         run: pytest python/tests

.gitignore

@@ -51,6 +51,8 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 .ruff_cache/
+bigquery.json
+metastore_db/
 
 # Translations
 *.mo
@@ -71,6 +73,9 @@ instance/
 # Sphinx documentation
 docs/_build/
 
+# Mike Javadoc
+docs/javadoc
+
 # PyBuilder
 target/
 
@@ -128,3 +133,9 @@ target/
 
 # mkdocs intemediate files
 docs/generated
+
+docs/CONTRIBUTING.md
+docs/index.md
+
+# Test artifacts
+keyFile.json