Merge pull request #25 from nextflow-io/add-asciidoctor

Add initial documentation for the plugin and document assumptions for dev/local cluster

.github/workflows/ghpages.yml

@@ -0,0 +1,49 @@
+name: ghpages
+on:
+  push:
+    branches:
+      - 'main'
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Build docu
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Environment
+        run: env | sort
+      - uses: actions/checkout@v4
+      - name: Set up JDK for x64
+        uses: actions/setup-java@v3
+        with:
+          java-version: '19'
+          distribution: 'temurin'
+          architecture: x64
+      - name: Generate
+        run: ./gradlew asciidoctor
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          path: ./build/docs/asciidoc
+
+  publish-ghpages:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

build.gradle

@@ -1,3 +1,17 @@
+buildscript {
+    repositories {
+        mavenCentral()
+    }
+}
+
+plugins {
+    id("org.asciidoctor.jvm.convert") version "3.3.2"
+}
+
+repositories {
+    mavenCentral()
+}
+
 import nextflow.gradle.plugins.NextflowPlugin
 
 apply plugin: NextflowPlugin
@@ -14,3 +28,8 @@ nextflowPlugin{
     extensionPoints = [
     ]
 }
+
+asciidoctor{
+    baseDir 'src/docs/asciidoc'
+    attributes revnumber : { project.version.toString() }
+}

plugins/nf-nomad/src/main/nextflow/nomad/NomadHelper.groovy

@@ -1,6 +1,6 @@
 /*
- * Copyright 2023, Stellenbosch University, South Africa
- * Copyright 2022, Center for Medical Genetics, Ghent
+ * Copyright 2023-, Stellenbosch University, South Africa
+ * Copyright 2024, Evaluacion y Desarrollo de Negocios, Spain
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy

@@ -1,6 +1,6 @@
 /*
- * Copyright 2023, Stellenbosch University, South Africa
- * Copyright 2022, Center for Medical Genetics, Ghent
+ * Copyright 2023-, Stellenbosch University, South Africa
+ * Copyright 2024, Evaluacion y Desarrollo de Negocios, Spain
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

plugins/nf-nomad/src/test/nextflow/nomad/NomadConfigSpec.groovy

@@ -1,4 +1,6 @@
 /*
+ * Copyright 2023-, Stellenbosch University, South Africa
+ * Copyright 2024, Evaluacion y Desarrollo de Negocios, Spain
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

plugins/nf-nomad/src/test/nextflow/nomad/TestHelper.groovy

@@ -1,6 +1,6 @@
 /*
- * Copyright 2020-2022, Seqera Labs
- * Copyright 2013-2019, Centre for Genomic Regulation (CRG)
+ * Copyright 2023-, Stellenbosch University, South Africa
+ * Copyright 2024, Evaluacion y Desarrollo de Negocios, Spain
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadServiceSpec.groovy

@@ -1,4 +1,6 @@
 /*
+ * Copyright 2023-, Stellenbosch University, South Africa
+ * Copyright 2024, Evaluacion y Desarrollo de Negocios, Spain
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadTaskHandlerSpec.groovy

@@ -1,4 +1,6 @@
 /*
+ * Copyright 2023-, Stellenbosch University, South Africa
+ * Copyright 2024, Evaluacion y Desarrollo de Negocios, Spain
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package nextflow.nomad.executor
 
 import nextflow.exception.ProcessSubmitException

src/docs/asciidoc/index.adoc

@@ -0,0 +1,8 @@
+= nf-nomad
+:toc: left
+
+include::parts/intro.adoc[]
+
+include::parts/development.adoc[]
+
+include::parts/usage.adoc[]

src/docs/asciidoc/parts/development.adoc

@@ -0,0 +1,106 @@
+== Development
+
+Firstly you need a `nomad` cluster. See https://developer.hashicorp.com/nomad/intro
+
+In this setup, we have made the following assumptions
+
+- You want to create a minimal `Nomad cluster` for evaluation purpose
+- You have access to an AWS S3-compatible storage or GCP Storage
+- Your linux instance already has `docker` installed
+
+==== Setup local nomad dev cluster with FusionFS
+
+Here's a sample Nomad client configuration, please refer the official https://developer.hashicorp.com/nomad/docs/install[installation ] instructions and https://developer.hashicorp.com/nomad/tutorials/get-started/gs-start-a-cluster[dev cluster] instructions for customizing this for your own linux machine.
+
+.nomad.hcl
+[source,hcl]
+----
+# Full configuration options can be found at https://www.nomadproject.io/docs/configuration
+
+data_dir  = "/opt/nomad/data"
+datacenter = "dc"
+
+server {
+  enabled          = true
+  bootstrap_expect = 1
+}
+
+client {
+  enabled = true
+
+  options = {
+    "driver.allowlist" = "docker,raw_exec"
+  }
+
+  host_volume "scratch" {
+    path      = "/home/<USERNAME>/_nomad_volume_scratch"
+    read_only = false
+  }
+
+}
+
+acl {
+  enabled = true
+}
+
+
+plugin "raw_exec" {
+  config {
+    enabled = true
+  }
+}
+
+
+plugin "docker" {
+  config {
+    allow_privileged = true
+
+    volumes {
+      enabled      = true
+    }
+}
+
+----
+
+
+[source,bash]
+----
+$ sudo nomad agent -dev \
+  -bind 0.0.0.0 \
+  -network-interface='{{ GetDefaultInterfaces | attr "name" }}'
+
+$ export NOMAD_ADDR=http://localhost:4646
+----
+
+
+==== Build and use nf-nomad plugin
+
+If you'd like to contribute to the plugin (yay!), please follow these instructions.
+
+TIP: As of 08-MARCH-2024, we are using Java-19 for the gradle build tasks. We recommend that you setup your Java SDK accordingly.
+
+1. Clone the repository
+
+[source,bash]
+----
+$ git clone https://github.com/nextflow-io/nf-nomad.git
+----
+
+2. Open the project in `IntelliJ`
+
+3. Make your changes to the plugin codebase
+
+4. Run the following gradle tasks on the command line from the project root (or via IntelliJ)
+
+[source,bash]
+----
+$ ./gradlew clean zipPlugin
+----
+
+5. Install the locally built plugin to the `$HOME/.nextflow/plugins` folder
+
+[source,bash]
+----
+$ ./gradlew unzipPlugin
+----
+

src/docs/asciidoc/parts/intro.adoc

@@ -0,0 +1,28 @@
+== Introduction
+
+The aim of this plugin is to integrate `Nextflow` and Hashicorp `Nomad` executor for batch processing of jobs.
+
+The plugin itself, is similar in design and scrope to other executor integrations such as `nf-azure` and `k8s` integration, however as `Nomad` (arguably) offers a simpler setup, we aimed to address the usability of a Nomad cluster within the Nextflow ecosystem.
+
+Hope you enjoy using the plugin and please participate in the development/feedback of the plugin in `platform-nomad` channel on Nextflow Slack instance. We would love to hear your feedback!
+
+
+
+==== Some resources for familiarizing yourself with Nomad:
+
+-  https://developer.hashicorp.com/nomad/docs?product_intent=nomad[Hashicorp Documentation website for Nomad]
+
+- https://developer.hashicorp.com/nomad/docs/configuration[Nomad Client/Server Configuration]
+
+- https://developer.hashicorp.com/nomad/docs/job-specification[Nomad Job Specification]
+
+- https://developer.hashicorp.com/nomad/docs/other-specifications/volume[Nomad Volumes specification]
+
+- https://developer.hashicorp.com/nomad/docs/runtime[Nomad Runtime]
+
+- https://developer.hashicorp.com/nomad/docs/drivers/docker[Task Driver - Docker]
+
+- https://developer.hashicorp.com/nomad/plugins/drivers/podman[Task Driver - Podman]
+
+- https://developer.hashicorp.com/nomad/docs/drivers/exec[Task Driver Isolated Fork/Exec ]
+

src/docs/asciidoc/parts/usage.adoc

@@ -0,0 +1,70 @@
+== Usage
+
+Using the `nf-nomad` plugin is relatively straigtforward, once you have access to a development or production `nomad cluster`. Refer development.adoc for creating a local development cluster on a linux machine.
+
+Assuming that you have access to a minimal `Nomad cluster` and would like to evaluate the `nomad` executor and the `nf-nomad` plugin,  you can rely upon Seqera Labs innovative Fusion FileSystem with a
+
+.nextflow.config
+[source,groovy]
+----
+
+// add the relevant plugins
+plugins {
+  id 'nf-nomad'
+  id 'nf-amazon'
+}
+
+// enable the fusion file system and wave service to facilitate fusionfs
+fusion {
+    enabled = true
+    exportstoragecredentials=true
+}
+
+wave {
+    enabled = true
+}
+
+// specify nomad executor and configuration
+process {
+    executor ='nomad'
+}
+
+nomad{
+    client{
+        address = "http://localhost:4646"
+    }
+    jobs{
+        deleteoncompletion = false
+    }
+}
+
+
+----
+
+.main.nf
+[source,groovy]
+----
+process sayHello {
+    container   'ubuntu:20.04'
+
+    input:
+    val x
+    output:
+    stdout
+    script:
+    """
+    echo '$x world!'
+    """
+}
+
+workflow {
+    Channel.of('Bonjour', 'Ciao', 'Hello', 'Hola') | sayHello | view
+}
+----
+
+TIP:: You need to use an S3 bucket as working dir
+
+Run the pipeline
+
+`$ nextflow run main.nf`
+