Prepare updates for v0.0.4 (#121)

* re-align to the correct main

.dockerignore

@@ -0,0 +1 @@
+docker/test/Dockerfile

.github/workflows/build_images.yaml

@@ -34,6 +34,7 @@ jobs:
           outputs: "type=registry,push=true"
           tags: |
             ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/virtual-kubelet-inttw:${{ env.RELEASE_VERSION }}
+            ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/virtual-kubelet-inttw:latest
           file: ./docker/Dockerfile.vk
           platforms: linux/amd64, linux/arm64
 
@@ -66,5 +67,6 @@ jobs:
           outputs: "type=registry,push=true"
           tags: |
             ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/virtual-kubelet-inttw-refresh:${{ env.RELEASE_VERSION }}
+            ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/virtual-kubelet-inttw-refresh:latest
           file: ./docker/Dockerfile.refresh-token
           platforms: linux/amd64, linux/arm64

.gitignore

@@ -15,5 +15,5 @@ serviceaccount.yaml
 .knoc
 .tmp
 kustomizations_tmp
-ca*
-condor*
+ca.*
+condor*

README.md

@@ -1,123 +1,40 @@
-![Interlink logo](./imgs/interlink_logo.png)
+![Interlink logo](./docs/imgs/interlink_logo.png)
 
 ## :information_source: Overview
 
 InterLink aims to provide an abstraction for the execution of a Kubernetes pod on any remote resource capable of managing a Container execution lifecycle.
 
-__Goals__:
-- TBD
-
-__Non-goals__:
-- TBD
-
 The project consists of two main components:
 
 - __A Kubernetes Virtual Node:__ based on the [VirtualKubelet](https://virtual-kubelet.io/) technology. Translating request for a kubernetes pod execution into a remote call to the interLink API server.
 - __The interLink API server:__ a modular and pluggable REST server where you can create your own Container manager plugin (called sidecars), or use the existing ones: remote docker execution on a remote host, singularity Container on a remote SLURM batch system.
 
-The project is reusing and get inspired by the [KNoC](https://github.com/CARV-ICS-FORTH/knoc) boilerplates at different levels, implementing the generic API layer in place of an ssh execution as the original contribution.
-
-
-![drawing](imgs/InterLink.svg)
-
-This repository includes:
-
-- License information
-- Copyright and author information
-- Code of conduct and contribution guidelines
-- Templates for PR and issues
-- Code owners file for automatic assignment of PR reviewers
-- [GitHub actions](https://github.com/features/actions) workflows for linting
-  and checking links
+The project got inspired by the [KNoC](https://github.com/CARV-ICS-FORTH/knoc) project, enhancing that with the implemention a generic API layer b/w the virtual kubelet component and the provider logic for the container lifecycle management.
 
-Content is based on:
-
-- [Contributor Covenant](http://contributor-covenant.org)
-- [Semantic Versioning](https://semver.org/)
-- [Chef Cookbook Contributing Guide](https://github.com/chef-cookbooks/community_cookbook_documentation/blob/master/CONTRIBUTING.MD)
+![drawing](docs/imgs/InterLink.svg)
 
 
 ## Quick references:
 - [:information\_source: Overview](#information_source-overview)
 - [Quick references:](#quick-references)
 - [:fast\_forward: Quick Start](#fast_forward-quick-start)
-  - [Virtual node setup](#virtual-node-setup)
-    - [:grey\_exclamation:  Requirements](#grey_exclamation--requirements)
-    - [Bring up the virtual node](#bring-up-the-virtual-node)
-  - [Setup a Dummy remote executer](#setup-a-dummy-remote-executer)
-  - [:wrench: Kustomizing your Virtual Kubelet](#wrench-kustomizing-your-virtual-kubelet)
-  - [Going serious with InterLink and Sidecars](#going-serious-with-interlink-and-sidecars)
-  - [:information\_source: InterLink Config file](#information_source-interlink-config-file)
-  - [:information\_source: Environment Variables list](#information_source-environment-variables-list)
+- [:information\_source: InterLink Config file](#information_source-interlink-config-file)
+- [:information\_source: Environment Variables list](#information_source-environment-variables-list)
 - [Digging in the project](#digging-in-the-project)
+  - [Virtual Kubelet](#virtual-kubelet)
+  - [InterLink](#interlink)
+  - [Sidecars](#sidecars)
+    - [Docker Sidecar](#docker-sidecar)
+    - [Slurm Sidecar](#slurm-sidecar)
 - [GitHub repository management rules](#github-repository-management-rules)
   - [Merge management](#merge-management)
   - [Protection on main branch](#protection-on-main-branch)
 
 ## :fast_forward: Quick Start
 
-### Virtual node setup
-
-#### :grey_exclamation:  Requirements
-
-- A working Kubernetes instance >1.24
-  - if you are in a hurry:
-    - `curl -sfL https://get.k3s.io |  INSTALL_K3S_VERSION=v1.25.11+k3s1 INSTALL_K3S_EXEC="--tls-san X.X.X.X" sh -s - --disable traefik --disable metric-server`
-      - you do need `--tls-san X.X.X.X` only in case of a machine with a Floating IP attached 
-      - `k3s kubect get node` to check whenever the cluster is ready
-- Kubectl
-- Docker 
-
-#### Bring up the virtual node
-
-Fastest way to start using interlink, is by deploying a VK in Kubernetes using the prebuilt image:
+Visit the SLURM batch system example in the [gallery](./examples/interlink-slurm). You will deploy on your laptop a fully emulated instance of a kubernetes cluster able to offload the execution of a pod to a SLURM cluster
 
-```bash
-kubectl create ns vk
-kubectl apply -n vk -k ./kustomizations
-```
-
-In `./kustomizations` you can then play with the different configuration and deployment files in order to customize your setup, as described [here](#wrench-kustomizing-your-virtual-kubelet) .
-
-### Setup a Dummy remote executer
-
-- Use Docker Compose to create and start up containers:
-    ```bash
-    docker compose -f docker-compose.yaml up -d
-    ```
-- You are now running:
-    - A Virtual Kubelet
-    - The InterLink service
-    - A Docker Sidecar
-- Submit a YAML to your K8S cluster to test it. You could try:
-    ```bash
-    kubectl apply -f examples/busyecho_k8s.yaml -n vk
-    ```
-You will see now a Container starting up on your host, but managed by the docker compose interlink daemons.
-
-### :wrench: Kustomizing your Virtual Kubelet
-
-To customize your virtual node deployment edit the configuration files within the kustomizations directory:
-- `kustomization.yaml`: here you can specify resource files and generate configMaps
-- `deployment.yaml`: that's the main file you want to edit. Nested into spec -> template -> spec -> containers you can find these fields:
-    - name: the Container name
-    - image: Here you can specify which image to use, if you need another one. 
-    - args: These are the arguments passed to the VK binary running inside the Container.
-    - env: Environment Variables used by kubelet and by the VK itself. Check the ENVS list for a detailed explanation on how to set them.
-- `vk-cfg.json`: it's the config file for the VK itself. Here you can specify how many resources to allocate for the VK. Note that the name specified here for the VK must match the name given in the others config files.
-- `InterLinkConfig.yaml`: configuration file for the inbound/outbound communication (and not only) to/from the InterLink module. For a detailed explanation of all fields, check the [InterLink Config File](#information_source-interlink-config-file) section.
-If you perform any change to the listed files, you will have to
-```bash
-kubectl apply -n vk -k ./kustomizations
-```
-You can also use Environment Variables to overwrite the majority of default values and even the ones configured in the InterLink Config file. Check the [Environment Variables list](#information_source-environment-variables-list) for a detailed explanation.
-
-### Going serious with InterLink and Sidecars
-
-__You can find instructions on how to get started with installation script (itwinctl) [here](./docs/README.md).__
-
-
-### :information_source: InterLink Config file
+## :information_source: InterLink Config file
 Detailed explanation of the InterLink config file key values.
 | Key         | Value     |
 |--------------|-----------|
@@ -137,7 +54,7 @@ Detailed explanation of the InterLink config file key values.
 | TsocksPath | path to your tsocks library. |
 | TsocksLoginNode | specify an existing node to ssh to. It will be your "window to the external world" |
 
-### :information_source: Environment Variables list
+## :information_source: Environment Variables list
 Here's the complete list of every customizable environment variable. When specified, it overwrites the listed key within the InterLink config file.
 
 | Env         | Value     |
@@ -181,6 +98,7 @@ A quick recap to the list of outgoing HTTP calls:
 | Create        |    InterLinkUrl:InterLinkPort/create    |
 | Delete        |    InterLinkUrl:InterLinkPort/delete    |
 | Status        |    InterLinkUrl:InterLinkPort/status    |
+| Logs          |    InterLinkUrl:InterLinkPort/getLogs   |
 
 ### InterLink
 InterLink is the middleware in charge to translate Virtual Kubelet's HTTP calls in a standard, agnostic outputs understandable to whatever plugin, called in this context Sidecar, below him. To properly work, a working Kubeconfig must be provided by the Virtual Kubelet.  
@@ -206,6 +124,7 @@ A quick recap to the list of HTTP calls:
 | Create        |   InterLinkUrl:InterLinkPort/create   | SidecarURL:SidecarPort/create |
 | Delete        |   InterLinkUrl:InterLinkPort/delete   | SidecarURL:SidecarPort/delete |
 | Status        |   InterLinkUrl:InterLinkPort/status   | SidecarURL:SidecarPort/status |
+| Logs          |    InterLinkUrl:InterLinkPort/getLogs | SidecarURL:SidecarPort/logs   |
 
 ### Sidecars
 Sidecars are modular plugins able to talk to InterLink API and, so, to the Kubernetes cluster. They performs specific operations based on what the actual plugin is, but the idea behind them is to submit a Pod on the cluster and, according to annotations, mountpoints, secrets, etc, deploy specific services.
@@ -266,4 +185,3 @@ To be configured on the repository settings.
   - Other checks as available and relevant
   - Require branches to be up to date before merging
 - Include administrators
-

cmd/interlink/main.go

@@ -15,6 +15,8 @@ var Url string
 
 func main() {
 	var cancel context.CancelFunc
+
+	commonIL.NewInterLinkConfig()
 	logger := logrus.StandardLogger()
 
 	if commonIL.InterLinkConfigInst.VerboseLogging {
@@ -26,12 +28,9 @@ func main() {
 	}
 
 	log.L = logruslogger.FromLogrus(logrus.NewEntry(logger))
-
 	interlink.Ctx, cancel = context.WithCancel(context.Background())
 	defer cancel()
 
-	commonIL.NewInterLinkConfig()
-
 	log.G(interlink.Ctx).Info(commonIL.InterLinkConfigInst)
 
 	mutex := http.NewServeMux()

cmd/sidecars/docker/main.go

@@ -16,6 +16,8 @@ func main() {
 	var cancel context.CancelFunc
 	logger := logrus.StandardLogger()
 
+  commonIL.NewInterLinkConfig()
+
 	if commonIL.InterLinkConfigInst.VerboseLogging {
 		logger.SetLevel(logrus.DebugLevel)
 	} else if commonIL.InterLinkConfigInst.ErrorsOnlyLogging {
@@ -29,8 +31,6 @@ func main() {
 	docker.Ctx, cancel = context.WithCancel(context.Background())
 	defer cancel()
 
-	commonIL.NewInterLinkConfig()
-
 	mutex := http.NewServeMux()
 	mutex.HandleFunc("/status", docker.StatusHandler)
 	mutex.HandleFunc("/create", docker.CreateHandler)

cmd/sidecars/slurm/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"net/http"
+	"strconv"
 
 	commonIL "github.com/intertwin-eu/interlink/pkg/common"
 	slurm "github.com/intertwin-eu/interlink/pkg/sidecars/slurm"
@@ -15,6 +16,8 @@ func main() {
 	var cancel context.CancelFunc
 	logger := logrus.StandardLogger()
 
+	commonIL.NewInterLinkConfig()
+
 	if commonIL.InterLinkConfigInst.VerboseLogging {
 		logger.SetLevel(logrus.DebugLevel)
 	} else if commonIL.InterLinkConfigInst.ErrorsOnlyLogging {
@@ -24,18 +27,20 @@ func main() {
 	}
 
 	log.L = logruslogger.FromLogrus(logrus.NewEntry(logger))
+	log.G(context.Background()).Debug("Debug level: " + strconv.FormatBool(commonIL.InterLinkConfigInst.VerboseLogging))
 
 	slurm.Ctx, cancel = context.WithCancel(context.Background())
 	defer cancel()
 
-	commonIL.NewInterLinkConfig()
-
 	mutex := http.NewServeMux()
 	mutex.HandleFunc("/status", slurm.StatusHandler)
 	mutex.HandleFunc("/create", slurm.SubmitHandler)
 	mutex.HandleFunc("/delete", slurm.StopHandler)
 	mutex.HandleFunc("/getLogs", slurm.GetLogsHandler)
 
+	slurm.CreateDirectories()
+	slurm.Load_JIDs()
+
 	err := http.ListenAndServe(":"+commonIL.InterLinkConfigInst.Sidecarport, mutex)
 	if err != nil {
 		log.G(slurm.Ctx).Fatal(err)

docker/Dockerfile.interlink

@@ -1,4 +1,4 @@
-FROM golang:1.19 as build-stage
+FROM golang:1.21 as build-stage
 
 WORKDIR /app
 
@@ -7,15 +7,12 @@ COPY .. .
 RUN CGO_ENABLED=0 GOOS=linux go build -mod vendor -o bin/interlink cmd/interlink/main.go
 
 # Deploy the application binary into a lean image
-FROM gcr.io/distroless/base-debian11 AS build-release-stage
+FROM gcr.io/distroless/base-debian11:latest AS build-release-stage
 
 WORKDIR /
 
 COPY --from=build-stage /app/bin/interlink /interlink
 
-COPY ../kustomizations/InterLinkConfig.yaml .
-ENV INTERLINKCONFIGPATH=/InterLinkConfig.yaml
-
 USER nonroot:nonroot
 
-ENTRYPOINT ["/interlink"]
+ENTRYPOINT ["/interlink"]

docker/Dockerfile.refresh-token

@@ -1,7 +1,7 @@
 FROM python:3.10
 
-RUN pip3 install requests
+RUN pip3 install --no-cache-dir requests==2.31.0
 
-COPY ../scripts/refresh.py /opt/refresh.py
+COPY ../docker/scripts/refresh.py /opt/refresh.py
 
 ENTRYPOINT ["python3", "/opt/refresh.py"]

docker/Dockerfile.vk

@@ -1,6 +1,6 @@
 FROM bitnami/kubectl as kubectl
 
-FROM golang:1.19 as build-stage
+FROM golang:1.21 as build-stage
 
 WORKDIR /app
 
@@ -9,12 +9,12 @@ COPY .. .
 RUN CGO_ENABLED=0 GOOS=linux go build -mod vendor -o bin/vk
 
 # Deploy the application binary into a lean image
-FROM ubuntu:latest AS build-release-stage
+FROM ubuntu:22.04 AS build-release-stage
 
 WORKDIR /
 
 COPY --from=build-stage /app/bin/vk /vk
 
 COPY --from=kubectl /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/
 
-ENTRYPOINT ["/vk"]
+ENTRYPOINT ["/vk"]

docker/slurm/.dockerignore

@@ -0,0 +1 @@
+Dockerfile