Merge pull request #51 from eth-cscs/docs

Merge readthedocs documentation

.gitignore

@@ -6,3 +6,4 @@ __pycache__/
 *.rdb
 **/minio/data/
 **/deploy/demo/minio/
+doc/build/

doc/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

doc/source/client.rst

@@ -0,0 +1,3 @@
+============
+Client setup
+============

doc/source/conf.py

@@ -0,0 +1,80 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'FirecREST'
+copyright = '2018-2020, ETH Zurich'
+author = 'CSCS'
+
+# The master toctree document.
+master_doc = 'index'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinxcontrib.openapi',
+    'sphinx_tabs.tabs',
+    'sphinxcontrib.redoc',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+redoc = [
+    {
+        'name': 'FirecREST API',
+        'page': 'api',
+        'spec': '../openapi/firecrest-developers-api.yaml',
+        'embed': True,
+        'opts': {
+            'lazy': False,
+            'nowarnings': True,
+            'nohostname': False,
+            'required-props-first': True,
+            # 'expand-responses': [200, 201],
+        }
+    },
+]

doc/source/index.rst

@@ -0,0 +1,29 @@
+Welcome to FirecREST
+=====================================
+
+FirecREST platform, a RESTful Services Gateway to High-Performance Computing (HPC) resources, is a high-performance and reusable framework that integrates with existing HPC infrastructure, thus enabling the access to HPC resources to web-enabled services.
+
+FirecREST provides a REST API that defines a set of HTTP methods through which developers can interact with using the HTTP/REST protocol architecture.
+Calls to the REST API received are translated into the appropriate infrastructure requests.
+Among the most prominent services that FirecREST exposes we find authentication and authorization, execution of parallel jobs through a workload manager, file-system operations, data mover, system status, system accounting information, etc.
+
+.. toctree::
+   :caption: Table of Contents:
+
+   Overview <overview>
+   Hands On <tutorial>
+   Reference Guide <reference>
+..   A python client setup <client>
+
+
+Latest Release
+==============
+
+FirecREST is being actively developed at `CSCS <http://www.cscs.ch/>`__.
+You can always find the latest release `here <https://github.com/eth-cscs/firecrest/releases/latest>`__.
+
+Publications
+============
+
+FA Cruz, M Martinasso. FirecREST: RESTful API on Cray XC systems [`pdf <https://arxiv.org/pdf/1911.13160.pdf>`__]
+

doc/source/overview.rst

@@ -0,0 +1,98 @@
+========
+Overview
+========
+
+FirecREST in a nutshell
+=======================
+
+FirecREST is a web-enabled application programming interface (API) to High-Performance Computing (HPC) resources under development at CSCS.
+
+Scientific platform developers can integrate the Firecrest API into their  web-enabled  portals  and  applications,  allowing  them  to securely  access  authenticated  and  authorized  CSCS  services such  as  job  submission,  data  mover  and  transfer  on  HPC systems.
+
+FirecREST architecture
+======================
+
+The core components and microservices that are part of FirecREST are the following:
+
+Gateway
+-------
+
+The API gateway provides an interface to publish, maintain, monitor, and secure all the FirecREST API endpoints.
+The gateway is hosted on a machine within CSCS that is facing the internet.
+All interactions with the FirecREST API are first passed and validated before being redirected to any other FirecREST microservice.
+In this way, every request made to the FirecREST API arrives first at the gateway, which will proxy the request towards the requested microservice endpoint.
+However, before the request is passed on to the microservice, the gateway will enforce that the request is correctly authenticated and authorized by requiring and validating the Access Token that must accompany each API request.
+The current implementation of the gateway service is based on the Kong API gateway.
+Kong is a widely used opensource microservice API gateway that implements functionalities such as a variety of authentication and authorization mechanisms, support for OIDC, IP filtering, access control lists, analytics, rate limiting, among many others that have allowed us to configure the gateway to our requirements.
+
+Status microservice
+-------------------
+
+Checks systems and services availability.
+The Status microservice provides information of infrastructure and services.
+
+Compute microservice
+--------------------
+
+The  compute microservice implements the interface to the workload manager, thus allowing applications to submit, stop, and  query  the  status  of jobs by using non-blocking asynchronous API calls.
+This service depends on: the `tasks microservice <#tasks-microservice>`__ that  provides  a  temporal resource that tracks the state of each call; and, the `delegation microservice <#delegation-microservice>`__ that issues a restricted SHH certificate that allows the execution of operations on behalf of  the  user.
+
+Storage microservice
+--------------------
+
+This microservice enable users the upload and downloadof large files to/from CSCS, while also enabling the movement  of  data  within  the  different  filesystems  available  on the  system .
+It does so by using non-blocking calls to high-performance storage services while immediately responding with a reference to a resource that tracks the state of the request.
+
+Tasks microservice
+------------------
+
+The task microservice responds to the need of managing the state of request that are being resolved asynchronously.
+One clear example for the need of this microservice can be observed in the asynchronous data transfer operations handled by the storage microservice, as otherwise some of those workflows would not be possible.
+As such, FirecREST microservices during  an  asynchronous  request  can  rapidly create and respond with a new task resource.
+The operational result of the request is then tracked as the originating microservice continuously updates the task as progress is being made.
+Thus, task resources allow a client to perform other activities while a FirecREST asynchronous tasks  are completed.
+
+Utilities microservice
+----------------------
+
+The utilities microservice provides **synchronous** execution of some basic linux commands.
+As calls to the utilities microservice are blocking operations, these have a timeoutand are not recursive.
+
+Delegation microservice
+-----------------------
+
+The delegation microservice is a FirecREST internal service that is not exposed to the user.
+This service takes avalid JWT access token as input and creates a short-lived SSH certificate to be used to user authentication.
+
+
+FirecREST IAM
+=============
+
+The  Identity  and  Access  Management  (IAM)  infrastructure at CSCS ensures that users and web applications have the  appropriate  permissions  to  access  resources  at  CSCS by  using  a  secure  protocol.
+From  the  whole  of  the  IAM infrastructure  at  CSCS  we  will  only  discuss  Keycloak,  the Identity  and  Access  Management  solution  deployed  at  thecenter.
+Keycloak  allows  to  secure  application  and  services by  providing  a  mechanism  for  the  authentication  and  authorization  of  CSCS  users,  CSCS  services,  and  third  party applications.
+
+Among  the  many  features  of  Keycloak  we highlight the following:
+
+- single sign-on solution
+- integration with Kerberos authentication service
+- fine-grained authorization controls for services
+- client registration and authorization
+- support of OpenID Connect (OIDC) protocol
+
+The  integration  of  FirecREST  with  Keycloak  has  been achieved  through  the  use  of  the  OIDC  protocol.
+OIDCis  an  authentication  protocol  that  extends  the  OAuth  2.0 specification.
+OAuth  2.0  is  an  industry-standard  protocol for  token-based  authorization  that  is  commonly  used  as  a mechanism  for  users  to  grant  access  permission  to  web application  in  order  to  access  user-owned  resources  and services.
+The  extensions  provided  by  OIDC  to  the  OAuth2.0  protocol  add  a  user  authentication  layer,  providing  a mechanism that enables single sign-on to users.
+FirecREST leverages on Keycloak and OIDC for authentication and authorization of web applications, enabling the following capabilities:
+
+- enforce that all requests are authenticated
+- applications never manipulate user credentials
+- only allow requests from registered applications
+- user-managed access permissions per application
+- stateless security model by use of tokens
+- short lifespan for sensitive access tokens
+- extended client sessions allowed through refresh tokens
+
+In a nutshell, FirecREST OIDC-based IAM enables theuser to login to a registered web application using their CSCS credentials and grant a web application with access to user-owned resource at the Center.
+Moreover, it does so without the user ever sharing their credentials with the web application.

doc/source/reference.rst

@@ -0,0 +1,13 @@
+===============
+Reference Guide
+===============
+
+.. note::
+    `This <api.html>`__ page could also be useful when the warnings are fixed.
+
+Includes all api calls from `firecrest-developers-api.yaml`.
+
+.. openapi:: ../openapi/firecrest-developers-api.yaml
+   :group:
+   :exclude:
+        /storage/xfer-external/upload-finished/{task_id}

doc/source/requirements.txt

@@ -0,0 +1,5 @@
+Sphinx==1.8.2
+sphinx-rtd-theme==0.4.3
+sphinxcontrib-openapi==0.6.0
+https://github.com/djungelorm/sphinx-tabs/releases/download/v1.1.13/sphinx-tabs-1.1.13.tar.gz
+sphinxcontrib-redoc==1.5.1