Merge pull request #1539 from thehyve/dev

Release version 1.0.1.

.github/workflows/build_and_deploy_docs.yaml

@@ -0,0 +1,104 @@
+name: Build and deploy docs to Github Pages
+
+on:
+  push:
+    branches:
+      - release
+
+jobs:
+  build-saturn:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Log details
+        run: |
+          BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
+          echo "Triggered on branch: $BRANCH"
+
+      - name: Set up Ruby (required for gem installation)
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3.2'
+
+      - name: Install Asciidoctor to build docs
+        run: |
+          gem install asciidoctor
+          gem install asciidoctor-pdf
+          gem install rouge
+
+      - name: Build Docs - Collect needed files
+        run: |
+          set -e
+          
+          # Initialize variables
+          PROJECT_FILES=(
+            "projects/saturn/src/main/resources/log4j2.properties"
+            "projects/saturn/src/main/resources/system-vocabulary.ttl"
+            "projects/saturn/taxonomies.ttl"
+            "projects/saturn/views.yaml"
+            "projects/saturn/vocabulary.ttl"
+          )
+          BUILD_DIR=./docs/build
+          version=$(cat VERSION)
+          
+          # Create build directory
+          mkdir -p $BUILD_DIR/docs
+          
+          # Copy all files to the build directory
+          cp ./README.adoc $BUILD_DIR
+          sed -i -e "s/VERSION/${version}/" $BUILD_DIR/README.adoc
+          cp -r ./docs/images $BUILD_DIR/docs/
+          for f in ${PROJECT_FILES[*]}; do
+            mkdir -p "$BUILD_DIR/$(dirname "$f")"
+            cp "$f" "$BUILD_DIR/"$(dirname "$f")""
+          done
+
+      - name: Build Docs - Generate PDF and HTML
+        run: |
+          set -e
+          
+          BUILD_DIR=./docs/build
+          
+          asciidoctor-pdf -a pdf-theme=./docs/pdf-theme.yml -o $BUILD_DIR/Fairspace.pdf $BUILD_DIR/README.adoc || {
+            echo "Error building PDF"
+            popd
+            exit 1
+          }
+          
+          asciidoctor -a toc=left -D $BUILD_DIR/ -o index.html $BUILD_DIR/README.adoc || {
+            echo "Error building site"
+            popd
+            exit 1
+          }
+          
+          rm $BUILD_DIR/README.adoc
+
+      - name: Deploy Docs (push to Github Pages, will be deployed automatically)
+        env:
+          CI_SERVICE_ACCOUNT_USER: ${{ secrets.CI_SERVICE_ACCOUNT_USER }}
+          CI_SERVICE_ACCOUNT_PASSWORD: ${{ secrets.FNS_PAT }}
+          DOCS_REPOSITORY_NAME: ${{ vars.DOCS_REPOSITORY_NAME }}
+        run: |
+          set -e
+          
+          DOCS_REPO_URL="https://${CI_SERVICE_ACCOUNT_USER}:${CI_SERVICE_ACCOUNT_PASSWORD}@github.com/thehyve/${DOCS_REPOSITORY_NAME}"
+          echo "Cloning documentation repository ${DOCS_REPOSITORY_NAME} ..."
+          git clone --branch main "${DOCS_REPO_URL}" fairspace-docs
+          
+          DOCS_DIR=$(pwd)/fairspace-docs
+          echo "Copying documentation to ${DOCS_DIR} ..."
+          cp -r ./docs/build/* "${DOCS_DIR}/"
+          cd "${DOCS_DIR}"
+          
+          if [ ! "$(git status -s)" == "" ]; then
+            echo "Committing and pushing changes to ${DOCS_REPOSITORY_NAME} ..."
+            git config  --global user.email "${CI_SERVICE_ACCOUNT_USER}@thehyve.nl"
+            git config  --global user.name "${CI_SERVICE_ACCOUNT_USER}"
+            git add .
+            git commit -a -m "Update documentation"
+            git push "${DOCS_REPO_URL}" main
+          else
+            echo "Documentation unchanged."
+          fi

README.adoc

@@ -1743,18 +1743,67 @@ Only allowed for administrators.
 | ``403``
 | Operation not allowed. The current user is not an administrator.
 | ``409``
+| Maintenance (reindexing or compacting) is already in progress.
+| ``503``
+| Service not available. This means that the application is configured not to use a view database.
+|===
+
+|===
+2+| ``POST /api/maintenance/compact``
+
+2+| Compact the Jena TDB database files.
+
+Jena database files grow fast when using transactions. This operation will compact the database files to reduce their size. If data is inserted using many small transactions the files will be reduced to 10-20% of their original size.
+
+Only allowed for administrators.
+2+| _Response:_
+| ``204``
+| Asynchronous task to compact Jena files has started.
+| ``403``
+| Operation not allowed. The current user is not an administrator.
+| ``409``
+| Maintenance (reindexing or compacting) is already in progress.
+| ``503``
+| Service not available. This means that the application is configured not to use a view database.
+|===
+
+
+.Example of compacting Jena using curl
+[%collapsible]
+====
+[source, bash]
+----
+curl -X POST 'http://localhost:8080/api/maintenance/compact'
+----
+====
+
+|===
+2+| ``GET /api/maintenance/status``
+
+2+| Get the status of maintenance tasks.
+
+It is not possible to run more than one maintenance task at the same time. If you start a task while another task is running, the new task will be rejected. If you want to know in advance whether a task is running, you can use this endpoint.
+
+A text is return
+
+2+| _Response:_
+| ``200``
+| Returns "active" or "inactive"
+| ``403``
+| Operation not allowed. The current user is not an administrator.
+| ``409``
 | Reindexing is already in progress.
 | ``503``
 | Service not available. This means that the application is configured not to use a view database.
 |===
 
 
-.Example recreate index using curl
+.Example of getting the maintenance status using curl
 [%collapsible]
 ====
 [source, bash]
 ----
-curl -X POST 'http://localhost:8080/api/maintenance/reindex'
+curl -X POST 'http://localhost:8080/api/maintenance/status'
 ----
 ====
 
@@ -1900,7 +1949,8 @@ Multiple external Fairspace metadata pages can be configured simultaneously. A l
 | String to be used as a display name of the metadata source.
 | ``url``
 | Fairspace instance to connect to. If the url is not specified, the metadata source will be treated as the internal one.
-| *Important!* There should only be a single configuration of internal metadata (only the first one will not be ignored).
+
+*Important!* There should only be a single configuration of internal metadata (only the first one will not be ignored).
 | ``icon-name``
 | Name of an icon configured in the "icons" section of values.yaml file. If the name is not specified, there will be a default icon used.
 |===
@@ -2498,18 +2548,20 @@ DOCKER_LOGGING_DRIVER=json-file
 ----
 
 To run the development version, checkout this repository,
-navigate to ``projects/mercury`` and run
+navigate to ``projects/mercury`` and run the following commands (``yarn install`` only has to be ran the first time running fairspace).
 
 [source, shell]
 ----
+yarn install
 yarn dev
 ----
 
+
 This will start a Keycloak instance for authentication at port ``5100``,
 the backend application named Saturn at port ``8080`` and the
 user interface at port ``3000``.
 
-At first run, you need to configure the service account in Keycloak.
+At first run, you need to configure the service account in Keycloak. If you cannot log in, you might need to restart fairspace by closing it and running ``yarn dev`` again.
 
 * Navigate to link:http://localhost:5100[http://localhost:5100]
 * Login with credentials ``keycloak``, ``keycloak``
@@ -2518,7 +2570,7 @@ At first run, you need to configure the service account in Keycloak.
 ** Click `Clients` in the left menu -> Select 'workspace-client'
 ** Choose tab `Service Account Roles`
 ** Click `Assign Role`
-** Select `Filter by clients` from the drop down menu and search for role name `view-users`. Then click `Asign role`.
+** Select `Filter by clients` from the drop down menu and search for role name `view-users`. Then click `Assign`.
 
 Now everything should be ready to start using Fairspace:
 
@@ -2633,7 +2685,7 @@ Create DNS records for the ``keycloak.example.com``, ``fairspace.example.com`` a
 # List available fairspace chart versions
 ~/bin/helm/helm search repo --versions fairspace/fairspace
 # Fetch the fairspace chart
-~/bin/helm/helm pull fairspace/fairspace --version 1.0.0
+~/bin/helm/helm pull fairspace/fairspace --version 1.0.1
 ----
 
 ===== Deploy Keycloak
@@ -2721,7 +2773,7 @@ Create a new deployment (called _release_ in helm terminology) and
 install the Fairspace chart:
 [source, shell]
 ----
-~/bin/helm/helm install fairspace-new fairspace/fairspace --version 1.0.0 --namespace fairspace-new \
+~/bin/helm/helm install fairspace-new fairspace/fairspace --version 1.0.1 --namespace fairspace-new \
 -f /path/to/values.yaml --set-file saturn.vocabulary=/path/to/vocabulary.ttl --set-file saturn.views=/path/to/views.yaml
 ----
 You can pass values files with ``-f`` and provide a file for a specified
@@ -2818,7 +2870,7 @@ Additionally, to include custom icons for `fairspace.icons` option, you need to
 
 [source, shell]
 ----
-~/bin/helm/helm install fairspace-new fairspace/fairspace --version 1.0.0 --namespace fairspace-new \
+~/bin/helm/helm install fairspace-new fairspace/fairspace --version 1.0.1 --namespace fairspace-new \
 -f /path/to/values.yaml --set-file saturn.vocabulary=/path/to/vocabulary.ttl --set-file saturn.views=/path/to/views.yaml --set-file svgicons.extra-icon=/path/to/extra-icon.svg
 ----
 
@@ -2874,7 +2926,7 @@ jupyterhub:
 To update a deployment using a new chart:
 [source, shell]
 ----
-~/bin/helm/helm upgrade fairspace-new fairspace-1.0.0.tgz
+~/bin/helm/helm upgrade fairspace-new fairspace-1.0.1.tgz
 ----
 With ``helm upgrade`` you can also pass new values files with ``-f``
 and pass files with ``--set-file`` as for ``helm install``.
@@ -3268,7 +3320,7 @@ To address this issue, materialized views are used for enhanced performance.
 There are two types of materialized views: one for denormalized data, which includes the view ID and attribute data of Set/TermSet types, and another for joined views.
 For each joined view, there is one corresponding join materialized view (as specified in the views.yaml config).
 
-Materialized views are refreshed during database reindexing and metadata updates, provided that the doViewsUpdate flag is set to true in the metadata endpoints.
+Materialized views are refreshed during database reindexing, on Saturn initialization stage and metadata updates, provided that the doViewsUpdate flag is set to true in the metadata endpoints. The refresh is performed concurrently what allows for the system to be available during the update providing the old version of data until the new one is ready. To skip materialized views refresh on Saturn initialization stage, update the Saturn ConfigMap setting false value to `viewDatabase.mvRefreshOnStartRequired`.
 
 ==== Extra file storage
 

VERSION

@@ -1 +1 @@
-1.0.0
+1.0.1

charts/fairspace/templates/project/configmap-saturn.yaml

@@ -31,6 +31,7 @@ data:
       blobStorePath: "/data/saturn/files/blobs"
     viewDatabase:
       enabled: true
+      mvRefreshOnStartRequired: true
     features:
 {{ toYaml .Values.fairspace.features | indent 6 }}
 {{ if has "ExtraStorage" .Values.fairspace.features }}

projects/saturn/application.yaml

@@ -55,6 +55,7 @@ viewDatabase:
   autoCommit: false
   maxPoolSize: 50
   connectionTimeout: 1000
+  mvRefreshOnStartRequired: true
 search:
   pageRequestTimeout: 10000
   countRequestTimeout: 60000

projects/saturn/src/main/java/io/fairspace/saturn/config/Config.java

@@ -115,6 +115,7 @@ public static class ViewDatabase {
         public int maxPoolSize = 50;
         public long connectionTimeout = 1000;
         public boolean autoCommit = false;
+        public boolean mvRefreshOnStartRequired = true;
     }
 
     public static class ExtraStorage {

projects/saturn/src/main/java/io/fairspace/saturn/rdf/SaturnDatasetFactory.java

@@ -22,9 +22,11 @@ public class SaturnDatasetFactory {
     /**
      * Returns a dataset to work with.
      * We're playing Russian dolls here.
-     * The original TDB2 dataset graph, which in fact consists of a number of wrappers itself (Jena uses wrappers everywhere),
+     * The original TDB2 dataset graph, which in fact consists of a number of
+     * wrappers itself (Jena uses wrappers everywhere),
      * is wrapped with a number of wrapper classes, each adding a new feature.
-     * Currently it adds transaction logging and applies default vocabulary if needed.
+     * Currently it adds transaction logging and applies default vocabulary if
+     * needed.
      */
     public static Dataset connect(Config.Jena config, ViewStoreClientFactory viewStoreClientFactory) {
         var restoreNeeded = isRestoreNeeded(config.datasetPath);
@@ -40,6 +42,7 @@ public static Dataset connect(Config.Jena config, ViewStoreClientFactory viewSto
         }
 
         if (restoreNeeded) {
+            log.warn("Jena restore is needed, starting automatic restore.");
             restore(dsg, txnLog);
         }
 

projects/saturn/src/main/java/io/fairspace/saturn/rdf/transactions/TxnIndexDatasetGraph.java

@@ -48,6 +48,10 @@ protected void onChange(QuadAction action, Node graph, Node subject, Node predic
         }
     }
 
+    public DatasetGraph getDatasetGraph() {
+        return dsg;
+    }
+
     @Override
     public void begin(TxnType type) {
         begin(TxnType.convert(type));

projects/saturn/src/main/java/io/fairspace/saturn/rdf/transactions/TxnLogDatasetGraph.java

@@ -22,10 +22,12 @@ public class TxnLogDatasetGraph extends AbstractChangesAwareDatasetGraph {
 
     private final TransactionLog transactionLog;
     private volatile AccessToken user;
+    private DatasetGraph dsg;
 
     public TxnLogDatasetGraph(DatasetGraph dsg, TransactionLog transactionLog) {
         super(dsg);
         this.transactionLog = transactionLog;
+        this.dsg = dsg;
     }
 
     /**
@@ -46,6 +48,10 @@ protected void onChange(QuadAction action, Node graph, Node subject, Node predic
         });
     }
 
+    public DatasetGraph getDatasetGraph() {
+        return dsg;
+    }
+
     @Override
     public void begin(TxnType type) {
         begin(TxnType.convert(type));