ExecutionBroker.tex

\documentclass[11pt,a4paper]{ivoa}
\input tthdefs

\usepackage{xspace}
% Standard terms used throughout the document,
% defined as macro commands to maintain consistency
% and avoid repeated spelling mistakes.

% Using non-breaking space character.
% https://stackoverflow.com/a/1012891

\usepackage[super]{nth}

\newcommand{\xml} {XML}
\newcommand{\json} {JSON}
\newcommand{\yaml} {YAML}
\newcommand{\http} {HTTP}
\newcommand{\rest} {REST}

\newcommand{\uuid} {UUID}

\newcommand{\openapi} {OpenAPI}
\newcommand{\datamodel} {data~model}
\newcommand{\webservice} {web service}
\newcommand{\webbrowser} {web browser}

\newcommand{\vo} {VO}
\newcommand{\vofull} {Virtual Observatory}
\newcommand{\ivoa} {IVOA}
\newcommand{\ivoafull} {International Virtual Observatory Alliance}
\newcommand{\uws} {UWS}
\newcommand{\vospace} {VOSpace}

\newcommand{\execworkerclass} {**ExecutionWorker**}
\newcommand{\execbrokerclass} {\textit{ExecutionBroker}}
\newcommand{\execbrokerservice}[1] {\textit{ExecutionBroker~service#1}}
\newcommand{\execbrokersession}[1] {\textit{ExecutionBroker~session#1}}
\newcommand{\execbrokerstate}[1] {\codeword{state#1}}

\newcommand{\execoffer}[1] {\textit{ExecutionBroker~offer#1}}
\newcommand{\execofferset}[1] {\textit{ExecutionBroker~offerset#1}}
\newcommand{\execsession}[1] {\textit{ExecutionBroker~session#1}}

\newcommand{\executionbroker} {\textit{Execution~Broker}}
\newcommand{\executionplanning} {\textit{Execution~Planning}}
\newcommand{\execplatform} {\textit{execution~platform}}

\newcommand{\executable} {\textit{executable}}
\newcommand{\executablething}[1] {\textit{executable~thing#1}}
\newcommand{\excutabletask} {\textit{executable} task}

\newcommand{\metadoc} [1]{\textit{metadata document#1}}

%\newcommand{\execoffer}[1] {\textit{offer#1}}
\newcommand{\workerjob}[1] {\textit{session#1}}
\newcommand{\teardown} {tear-down}

\newcommand{\jupyter} {Jupyter}
\newcommand{\jupyterhub} {JupyterHub}
\newcommand{\binderhub} {BinderHub}
\newcommand{\jupyternotebook} {Jupyter notebook}

\newcommand{\esap} {ESAP}
\newcommand{\escape} {ESCAPE}
\newcommand{\datalake} {DataLake}
\newcommand{\rucio} {Rucio}

\newcommand{\python} {Python}
\newcommand{\pythonprogram} {Python program}
\newcommand{\pythonruntime} {Python runtime}

\newcommand{\apache} {Apache}
\newcommand{\spark} {Spark}
\newcommand{\pyspark} {PySpark}
\newcommand{\zeppelin} {Zeppelin}
\newcommand{\zeppelinnotebook} {Zeppelin notebook}

\newcommand{\oci} {OCI}
\newcommand{\ociruntime} {OCI runtime}
\newcommand{\ocicontainer} {OCI container}
\newcommand{\docker} {Docker}
\newcommand{\dockercompose} {Docker compose}
\newcommand{\dockerimage} {Docker image}
\newcommand{\dockerruntime} {Docker runtime}
\newcommand{\dockercontainer} {Docker container}

\newcommand{\singularity} {Singularity}
\newcommand{\singularitycontainer} {Singularity container}

\newcommand{\openstack} {Openstack}
\newcommand{\kubernetes} {Kubernetes}

\newcommand{\codeword}[1] {\texttt{#1}}
\newcommand{\footurl}[1] {\footnote{\url{#1}}}

\newcommand{\dataset}[1] {dataset#1}
\newcommand{\datascience} {data~science}
\newcommand{\scienceplatform}[1] {science~platform#1}

\newcommand{\science}[1] {science#1}
\newcommand{\scientist}[1] {scientist#1}

\newcommand{\cpu}[1] {CPU#1}
\newcommand{\gpu}[1] {GPU#1}
\newcommand{\nvidiagpu} {NVIDIA~AD104~GPU}

\newcommand{\scalable} {scalable}

% TODO add a citation for the YAML specification.
% https://yaml.org/spec/

\usepackage{listings}
\usepackage{xcolor}

%\colorlet{punct}{red!60!black}
\colorlet{numb}{magenta!60!black}
\definecolor{html-gray}{HTML}{EEEEEE}
\definecolor{light-gray}{gray}{0.95}
\definecolor{delim}{RGB}{20,105,176}

\lstset{
    basicstyle=\small\ttfamily,
    columns=fullflexible,
    frame=none,
    backgroundcolor=\color{light-gray},
    stepnumber=1,
    %numbers=left,
    numbers=none,
    numberstyle=\small,
    numbersep=8pt,
    %xleftmargin=\parindent,
    xrightmargin=1cm,
    showstringspaces=false,
    keepspaces=true,
    breaklines=true,
    linewidth=14cm,
    frame=none
}

% https://tex.stackexchange.com/questions/83085/how-to-improve-listings-display-of-json-files
% https://tex.stackexchange.com/a/83100
% https://tex.stackexchange.com/questions/10828/indent-a-code-listing-in-latex
% https://tex.stackexchange.com/a/10831
\lstdefinelanguage{json}{
    literate=
     *{0}{{{\color{numb}0}}}{1}
      {1}{{{\color{numb}1}}}{1}
      {2}{{{\color{numb}2}}}{1}
      {3}{{{\color{numb}3}}}{1}
      {4}{{{\color{numb}4}}}{1}
      {5}{{{\color{numb}5}}}{1}
      {6}{{{\color{numb}6}}}{1}
      {7}{{{\color{numb}7}}}{1}
      {8}{{{\color{numb}8}}}{1}
      }

\lstdefinelanguage{yaml}{
    literate=
     *{0}{{{\color{numb}0}}}{1}
      {1}{{{\color{numb}1}}}{1}
      {2}{{{\color{numb}2}}}{1}
      {3}{{{\color{numb}3}}}{1}
      {4}{{{\color{numb}4}}}{1}
      {5}{{{\color{numb}5}}}{1}
      {6}{{{\color{numb}6}}}{1}
      {7}{{{\color{numb}7}}}{1}
      {8}{{{\color{numb}8}}}{1}
      }

\hyphenation{Exe-cut-able-Thing}

% Enable [*bold*] inside listings
% https://stackoverflow.com/a/24838471
% https://www.mrunix.de/forums/archive/index.php/t-42976.html
% https://en.wikibooks.org/wiki/LaTeX/Source_Code_Listings
% https://mirror.ox.ac.uk/sites/ctan.org/macros/latex/contrib/listings/listings.pdf#subsection.3.3
\renewcommand{\ttdefault}{pcr}
\lstset{moredelim=[is][\bfseries]{[*}{*]}}
\lstset{moredelim=[is][\itshape]{[+}{+]}}

\title{IVOA Execution Broker}

% see ivoatexDoc for what group names to use here; use \ivoagroup[IG] for
% interest groups.
\ivoagroup{GWS}

\author[http://www.ivoa.net/twiki/bin/view/IVOA/DaveMorris]
       {Dave Morris}
\author[http://www.ivoa.net/twiki/bin/view/IVOA/SaraBertocco]
       {Sara Bertocco}

\editor[http://www.ivoa.net/twiki/bin/view/IVOA/DaveMorris]
       {Dave Morris}

% \previousversion[????URL????]{????Concise Document Label????}
\previousversion{This is the first public release}

\begin{document}
\begin{abstract}
\label{abstract}

One of the long term goals of the \ivoa{} has been to enable users to
move the code to the data.
This is becoming more and more important as the size and complexity
of the \dataset{s} available in the virtual observatory increases.
%\citep{gaia-at-esac}
%\footurl{https://www.skao.int/en/explore/big-data}
%\footurl{https://www.lsst.org/scientists/keynumbers}

The \ivoa{} \executionbroker{} provides a step towards making this possible.

The \ivoa{} \executionbroker{} is designed to address a specific question;
given an executable thing, e.g. a \pythonprogram{} or \jupyternotebook{}.
What facilities are available to run it?

To do this, the \ivoa{} \executionbroker{} specification defines
a \datamodel{} and \webservice{} API for describing executable things
and the resources needed to execute them.

Together these components enable a user to ask a simple question
\textit{"Where (and when) can I execute my program?"}

This in turn enables users to move code between \scienceplatform{s}.
Allowing them to develop their code on one platform and then apply it to a different
\dataset{} by sending it to execute on another platform.

\end{abstract}

\section*{Acknowledgments}
\label{sect-acknowledgments}

The authors would like to thank all the participants in the IVOA and ESCAPE projects
who have contributed their ideas, critical reviews, and suggestions to this document.

\section*{Conformance-related definitions}
\label{sect-conformance}

The words ``MUST'', ``SHALL'', ``SHOULD'', ``MAY'', ``RECOMMENDED'', and
``OPTIONAL'' (in upper or lower case) used in this document are to be
interpreted as described in IETF standard RFC2119 \citep{std:RFC2119}.

The \emph{Virtual Observatory (VO)} is a general term for a collection of
federated resources that can be used to conduct astronomical research,
education, and outreach.
The \href{https://www.ivoa.net}{International Virtual Observatory Alliance (IVOA)}
is a global collaboration of separately funded projects to develop standards and
infrastructure that enable VO applications.

\section{Introduction}
\label{sect-introduction}

The \ivoa{} \executionbroker{} specification defines a \datamodel{} for describing executable tasks
and a \webservice{} interface for managing them.
Together these provide a common interface for service discovery, resource allocation
and execution scheduling across a heterogeneous federation of different types of
execution platform.

\begin{itemize}
    \item \execbrokerclass{} \datamodel{}  – a data model for describing execution sessions and their resource requirements.
    \item \execbrokerclass{} \webservice{} – a \rest{} based web service to find execution platforms, allocate resources and schedule execution sessions.
\end{itemize}

\subsection{Role within the VO Architecture}
\label{sub-ivoa-role}

% As of ivoatex 1.2, the architecture diagram is generated by ivoatex in
% SVG; copy ivoatex/archdiag-full.xml to role_diagram.xml and throw out
% all lines not relevant to your standard.
% Notes don't generally need this.  If you don't copy role_diagram.xml,
% you must remove role_diagram.pdf from SOURCES in the Makefile.
\begin{figure}
\centering
\includegraphics[width=0.9\textwidth]{role_diagram.pdf}
\caption{Architecture diagram showing the \ivoa{} \executionbroker{}'s role in the \ivoa}
\label{fig:archdiag}
\end{figure}

The \ivoa{} Architecture\citep{2010ivoa.rept.1123A} provides a high-level view of how \ivoa{}
standards work together to connect users and applications with providers of data
and services.
Fig.~\ref{fig:archdiag} shows the role the \ivoa{} \executionbroker{} plays within this architecture.

In response to the increasing size and complexity of the next generation of science \dataset{s}
a number of \ivoa{} members are developing intergrated \scienceplatform{s} which bring
together the \dataset{s} co-located with the compute resources needed to analyse
them.\footurl{https://data.lsst.cloud/}\footurl{https://rsp.lsst.io/}

These \scienceplatform{s} make extensive use of the \ivoa{} data models and
vocabularies to describe their \dataset{s}, and use the \ivoa{} data access
services to find and access data from other data providers.
In addition, some of the \scienceplatform{s} use \ivoa{} \vospace{} services to manage
data transfers to and from local storage co-located with the compute resources.

However, to date the \ivoa{} does not provide any APIs or services that
enable \scienceplatform{s} to exchange the software used to analyse the data.
The \ivoa{} \executionbroker{} provides a step towards making this possible.

This places the \ivoa{} \executionbroker{} in the same region of the \ivoa{} architecture
as the \ivoa{} \vospace{} specification \citep{2009ivoa.specQ1007G},
providing an infrastructure level service that enables service discovery,
resource allocation and execution scheduling across a heterogeneous federation
of execution platforms.

\subsection{Supplementary documents}
\label{sub-supplementary-documents}

\subsubsection{\openapi{} specification}
\label{subsub-openapi-specification}

This document is designed to read in combination with the \openapi{}
\footurl{https://www.openapis.org/} \footurl{https://swagger.io/specification/}
specification published alongside this document.

The \openapi{} specification defines the technical details of the
\executionbroker{} \webservice{} interface and the schema for the
\executionbroker{} \datamodel{}.
This document compliments the technical specification by using use cases
and examples to describe the intended service behaviour and
explain the reasoning behind some of the design choices.

The two documents are intended to be read together.
The machine-readable \openapi{} spcification defining the \textit{what},
and the human-readable text document describing the \textit{why}.

The \openapi{} specification associated with this document is
published in the following files:
\begin{itemize}
    \item \codeword{openapi.yaml} - The main service specification, including
    the \executionbroker{} service API and the core data model.
    \item \codeword{messages.yaml} - a supplimentary data model for INFO, WARN,
    and DEBUG messages embedded in the service responses.
    \item \codeword{utils.yaml} - a supplimentary data model for re-usable
    components such as ISO date formats, min/max pairs, and name-value maps.
\end{itemize}

This text document may include small examples of \openapi{} schema to
explain specific points, but these are for information only. Unless otherwise
specified, the \openapi{} specification itself should be assumed to be the
definitive source and this text document should be considered as secondary.

\subsubsection{IVOA profiles}
\label{subsub-ivoa-profiles}

This specification refers to the following documents
to cover the details of how an \executionbroker{} \webservice{}
should use the following protocols and data formats:
\begin{itemize}
    \item The \ivoa{} \rest{} profile, describing a common pattern
    for \ivoa{} \webservice{s} that implement
    \rest{}\footurl{https://en.wikipedia.org/wiki/REST}
    webservices.
    \item The \ivoa{} \http{} profile, describing how \ivoa{} \webservice{s}
    should use aspects of the \http{} protocol, including \http{} parameters,
    message content, content negotiation, and \http{} return codes.
    \item The \ivoa{} error messages profile, describing a common format
    for \ivoa{} \webservice{s} to format error messages.
    \item The \ivoa{} \json{} profile, describing how \ivoa{} \webservice{s}
    should serialize message content using the
    \json{}\footurl{https://www.json.org/json-en.html} data format.
    \item The \ivoa{} \yaml{} profile, describing how \ivoa{} \webservice{s}
    should serialize message content using the
    \yaml{}\footurl{https://yaml.org/} data format.
    \item The \ivoa{} \xml{} profile, describing how \ivoa{} \webservice{s}
    should serialize message content using the
    \xml{}\footurl{https://www.w3.org/XML/} data format.
\end{itemize}

Unless otherwise specified, an \executionbroker{} \webservice{} implementation
should follow the guidelines outined in these documents.

\subsubsection{IVOA Single-Sign-On}
\label{subsub-ivoa-sso}

An \ivoa{} \executionbroker{} service MAY use parts of the
\ivoa{} Single-Sign-On standard\citep{2017ivoa.spec.0524T}
for authentication, and the
\ivoa{} Credential Delegation Protocol \citep{2010ivoa.spec.0218P}
for delegating credentials to other services.

\subsection{Executable things}
\label{sub-executablething}

To understand the problem that the \ivoa{} \executionbroker{} is trying to solve
it is useful to describe what an \executablething{} is in this context.
In general terms, this document refers to something that can be executed, or run,
as an \executable{}.

To explain what this means we can start with a science domain function that we want to perform.
For example, the mathematical concept of the square root of a number.
We can calculate the square root of a positive number using the Newton–Raphson
algorithm\footurl{https://en.wikipedia.org/wiki/Newton\%27s_method}
which produces successively closer approximations to the result.
However, in general case, this mathematical description of the algorithm would not be
considered to be an \executablething{}.

We can write a \pythonprogram{} to use this algorithm to calculate the square root of a number.
This is the first identifiable \executablething{} in our example.
To be able to use this \executablething{}, you would need a computing resource with the appropriate
hardware and software environment. In this case, a computing resource with the \python{} interpreter
installed along with the additional \python{} modules required by the program.
This environment is often referred to as the \python{} runtime.

In the context of \scienceplatform{s} and \datascience{}, a common pattern is to provide this environment
using a Docker\footurl{https://docs.docker.com/get-started/what-is-a-container/}
or OCI\footurl{https://opencontainers.org/} container
to package the \pythonprogram{} and \python{} runtime together as a single binary object.
This package, or container, is itself an \executablething{}. One which requires a different execution
environment than the original \pythonprogram{}.
The aim of containerization is to package software components together with all the libraries and dependencies
they need as a single binary object that interfaces with a standard execution environment,
referred to as the \textit{container runtime}.
To be able to use this \executablething{}, you would need a computing resource with the appropriate
hardware and software environment. In this case, a computing resource with the \docker{} or \ocicontainer{}
runtime installed.

We could also create a \jupyternotebook{} that demonstrates how to use our \pythonprogram{}.
This is the third \executablething{} in our example.
One which provides an interactive environment for the user to experiment with.
As before, to be able to use this \executablething{}, we would need a computing resource with
the appropriate hardware and software environment.
In this case, a computer with the \jupyternotebook{} platform installed along with all the \python{} modules
needed by our \pythonprogram{}.
In the context of \scienceplatform{s} and \datascience{}, a common pattern is to provide this environment as a \webservice{}
that allows the user to interact with the \jupyternotebook{} via a \webbrowser.

From one algorithm that implements a science domain function, we have created three different \executablething{s}.
A \pythonprogram{}, a \dockercontainer{} packaging the \pythonprogram{}, and an interactive \jupyternotebook{}
that demonstrates how to use the \pythonprogram{}.
Each of which requires a different computing environment to execute.
A basic \python{} runtime, the \dockerruntime{}, and a \jupyternotebook{} service.

We may also want to consider the data that we are applying the algorithm to and the compute resources that
will be needed to process it.
If we are running some small experiments to learn how to use the algorithm, then a basic computing
resource will probably be sufficient.
However, if we have a \dataset{} of ten million numbers that we want to process, then we may
need to consider adding extra storage to handle the input data and the results.
For a large \dataset{} it may also be worth using a \gpu{} to accelerate the calculation.

The \ivoa{} \executionbroker{} \datamodel{} provides a way to describe what each of these \executablething{s}
are and what resources are needed to execute them.
This can include things like number of \cpu{} cores and amount of memory it needs,
whether it needs a \gpu{}, the location of the input data, the storage space needed to perform
the calculation, and the storage space needed to save the results.

\section{Service interaction}
\label{sect-service-interaction}

The interaction between a user, the client application they are using, and the services available in the \vofull{}
can be described as a conversation to discover where, how, and when, an \executablething{} that the user
has chosen can be executed.

\subsection{Discovery services}
\label{sub-discovery-services}

The conversation starts at the discovery stage, where the user uses discovery services to
select the software and \dataset{s} that they want to work with.

\includegraphics[width=0.9\textwidth]{diagrams/data-discovery.pdf}

The detailed specification for the software and data discovery services are beyond the
scope of this document. However we can outline some general requirements for them.

In both cases, the discovery process should not depend on the technical details
of the software or the \dataset{s}, but on their science domain functionality and properties.

From a science user's perspective they want to be able to find software that implements
a particular clustering algorithm, or a \dataset{} that is indexed according to a particular
coordinate system.
The programming language the software is written in and the file format of the \dataset{}
are at best secondary criteria.
In our square root example, we would expect our user to use search terms like \textit{'square root'}
or \textit{'newton raphson'} to find the software they need.
We wouldn't expect them to start out looking for a \textit{'python'} or \textit{'docker'} as their key search terms.
Ideally, if the \executionbroker{} service functions as intended, a science user should not
need to know about programming languages, software packaging or file formats.
The \executionbroker{} service should hide as much as possible of the technical details,
enabling the science user to get on with science.

Another important consideration is these discovery services should be designed to be domain agnostic.
Meaning that it should be possible to swap out an astronomy based discovery service
for an equivalent biochemistry discovery service and although the domain specific
terms and vocabulary will be different, the techical details of the service interfaces
should be the same.

\subsubsection{Software discovery}
\label{subsub-software-discovery}

There are three main components involved in software discovery, the metadata schema for
describing the software, one or more search services, and the
repositories where the \executablething{s} are stored.

The vocabularies and schema need to be based on use cases that start by describing what the
\scientist{} wants to do, and from that derrive what software tools they would need, and what terms
they would naturally use to describe them.

The \ivoa{} semantics and data modelling working groups have a lot of experience developing
vocabularies and data models to descibe \science{} data products, and is well placed
to develop the vocabularies needed to descibe astronomy software.

It is important to keep in mind that the requirement is not to model the technical properties
of the software itself e.g. what programming language it is written in or who funded the development.
The important things to model are the search terms that a \scientist{} is most likley to use to try to
find the software they need.

The second component is a searchable database that acceopts a list of search terms and responds with a
list of \metadoc{s} that describe \executablething{s} that match the criteria.

Before we look in detail at the content of the \metadoc{s} it is worth looking at where the \metadoc{s} are
stored in relation to the search service and the repositories where the \executablething{s} are stored.

In one scenario, all of the components can be co-located by the same service.
The database of search terms, the \metadoc{s}, and the binary files containing the \executablething{s}
can all be hosted by the same service implementation.

TODO diagram

An alternative implementation could store them at different locations, using
existing off-the-shelf software and services to host them.

There are a number of widely available content managment systems, both commercial
and open source, that would be capable of implementing the database of search terms.

If the \metadoc{s} are stored in the same database, then the response from a
database search could contain the \metadoc{s} themselves.

TODO diagram
database, results, contain \metadoc{s} from database

Alternatively, the \metadoc{s} could be stored at a separate location,
in an online git repository for example,
and the database search response simply contains a list of URLs that
point to the individual \metadoc{s}.

TODO diagram
database, results, links to \metadoc{s} in external repositories

The third part of the set is the binary image of the \executablething{}.
In most cases it would probably make sense for the \metadoc{} to reference
the \executablething{} as a binary file stored in an external repository
rather than trying to include the \executablething{} as a binary blob in
the database.

TODO diagram
database, results, links to \metadoc{s} with links to images

The system can use standard cryptographic signatures and checksums to ensure the validity
of the \metadoc{s} and the binary images they refer to even when they are stored and accesed
via external \nth{3} party services.

In summary, there are two things that need to be standardised for a software discovery service:
\begin{itemize}
    \item The inputs to the discovery service, including the metadata vocabularies
        used to describe the software in terms that make sense to the \scientist{}
        looking for them. For example what algorithm it implements, the type of input data it
        operates on, and the type of results it generates.
    \item The outputs of the discovery service, including the \metadoc{s} defined by this
        specification, that describe the binary images that package the software
        as \executablething{s}.
\end{itemize}

The other components in the software discovery stack, the database of search terms, and
the storage and access services for the \metadoc{s} and binary images, do not need to be
standardised at this stage.

\subsubsection{Data discovery}
\label{subsub-data-discovery}

TODO - update this with reference to \ivoa{} data product type and how this links on the
executable parameters.
https://www.ivoa.net/rdf/product-type/2024-05-19/product-type.html

\subsection{Execution Broker}
\label{sub-execution-broker}

Once the user has identified the software and data resources that they want to use,
the client application brings together details of the \executablething{} the user
has selected, the compute, storage, and data resources they want to use,
along with a schedule describing when the user wants it to execute,
to create a \metadoc{} description of the \execsession{} the user wants
to execute.

\begin{lstlisting}[]
[+# ExecutionBroker OfferSet request.+]
executable:
  ....
resources:
  ....
schedule:
  ....
\end{lstlisting}

\subsubsection{OfferSet request}
\label{subsub-offerset-request}

The client sends the \metadoc{} description to one or more \execbrokerclass{}
services to ask if they can meet the requirements and execute the \execsession{}.

Each \execbrokerservice{} evaluates the request and responds with a top level
\codeword{YES|NO} answer, and if
the answer is \codeword{YES}, a list of one or more \execoffer{s} describing how
the requested \execsession{} could be executed on the platform(s) represented by
that \execbrokerservice{}.

%\begin{lstlisting}[]
%[+# ExecutionBroker OfferSet request/response.+]
%Request  - Can this platform execute <session> ?
%Response - YES, list of <offer>[]
%\end{lstlisting}

\includegraphics[width=0.9\textwidth]{diagrams/request-offers.pdf}


\subsubsection{Resource allocation}
\label{subsub-resource-allocation}

The \execbrokerservice{} is responsible for managing the resources
within the underlying \execplatform{} that it represents.

The \execbrokerservice{} is responsible for keeping track of
reservations for the resources that it grants to clients in
response to execution requests.

The details of how an \execbrokerservice{} implements this is
beyond the scope of this specification.

This specification defines a set of black-box behaviours
that an \execbrokerservice{} MUST implement.

\begin{itemize}
    \item Each \execoffer{} MUST meet the minimum criteria specified in the request.
    \item Each \execoffer{} MUST be within the capabilities of the underlying \execplatform{} to be able to provide.

    \item An \execbrokerservice{} MAY offer more resources than the minimum value specified in the request.
    \item An \execbrokerservice{} MAY offer more execution time beyond the minimum value specified in the request.

    \item An \execbrokerservice{} SHOULD NOT offer more resources than the maximum value specified in the request.
    \item An \execbrokerservice{} SHOULD NOT offer more execution time beyond the maximum value specified in the request.
\end{itemize}

This specification recognises a set of black-box behaviours that an \execbrokerservice{} SHOULD implement.

\begin{itemize}
    \item An \execoffer{} made by an \execbrokerservice{} is assumed to be made in good faith, with a reasonable probability of success.
    \item An \execbrokerservice{} is free to just say \codeword{NO} to a request, for whatever reason.
\end{itemize}

In addition, this specification recognises a number of situational and dispositional factors
that may influence how an \execbrokerservice{} allocates resources.

\begin{itemize}
    \item Different implementations may represent a range of different types of \execplatform{s}.
    \begin{itemize}
        \item Science platforms.
        \begin{itemize}
            \item CANFAR.
            \item Azimuth.
            \item SciServer.
        \end{itemize}
        \item Cloud platforms.
        \begin{itemize}
            \item Kubernetes platforms.
            \item Openstack platforms.
            \item Commercial cloud platforms.
        \end{itemize}
        \item Service platforms.
        \begin{itemize}
            \item Slurm services.
        \end{itemize}
    \end{itemize}

    \item Different implementations may be connected to a range of different sizes of \execplatform{s}.
    \begin{itemize}
        \item Large scale HPC systems.
        \item Large scale commercial platforms.
        \item Medium scale on-premises platforms.
        \item Small scale specialist platforms.
    \end{itemize}

    \item Different deployments may apply different algorithms to manage the allocation of resources.
    \begin{itemize}
        \item Maximise the amount of resources offered to users.
        \item Maximise the number of jobs a \execplatform{} can execute.
        \item Prioritise interactive sessions by offering precise start times.
        \item Prioritise batch processing by offering wide start ranges.
    \end{itemize}

    \item Different deployments may apply different policies to manage the allocation of resources.
    \begin{itemize}
        \item Required security criteria.
        \item Recognised identity providers.
        \item Required group membership policies.
        \item Local resource allocation policies.
        \item Local white lists of allowed executables.
    \end{itemize}
\end{itemize}

\subsubsection{Execution scheduling}
\label{subsub-execution-scheduling}

This is already covered in a differnt section ..

\subsubsection{Offerset response}
\label{subsub-offerset-response}

Each \execbrokerservice{} will respond to a request for offers with an \execofferset{}
containing some metadata about the \execofferset{} itself, followed by a list of \execoffer{}s
describing how the requested \execsession{} could be executed.

Each \execoffer{} in the list contains some metadata about the \execoffer{} itself,
including its UUID identifier and expiry time, followed by details of how and when the
\execsession{} would be executed.

\begin{lstlisting}[]
[+# ExecutionBroker OfferSet response.+]
result: YES
....
offers:

- uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face"
  state: 'OFFERED'
  expires: "2023-09-18T07:05:21"
  ....
  executable:
    ....
  resources:
    ....
  schedule:
    ....

- uuid: "2e16bf4c-7ff6-11ef-8412-4bc36fe2face"
  status: 'OFFERED'
  expires: "2023-09-18T07:05:21"
  ....
  executable:
    ....
  resources:
    ....
  schedule:
    ....
\end{lstlisting}

\subsubsection{Session lifecycle}
\label{subsub-session-lifecycle}

An \execbrokersession{} goes through the following lifecycle stages.

\begin{itemize}
    \item \codeword{OFFERED}    The \workerjob{} is being offered.
    \item \codeword{ACCEPTED}   The \workerjob{} has been accepted.
    \item \codeword{REJECTED}   The \workerjob{} offer has been rejected.
    \item \codeword{EXPIRED}    The \workerjob{} offer has expired.
    \item \codeword{WAITING}    The \workerjob{} is waiting to activate.
    \item \codeword{PREPARING}  The resources are being prepared.
    \item \codeword{READY}      The \workerjob{} is ready to execute.
    \item \codeword{RUNNING}    The \workerjob{} is executing.
    \item \codeword{RELEASING}  The resources are being released.
    \item \codeword{COMPLETED}  The \workerjob{} has completed.
    \item \codeword{CANCELLED}  The \workerjob{} has been cancelled.
    \item \codeword{FAILED}     The \workerjob{} has failed.
\end{itemize}

The first four \execbrokerstate{s} are interactive, where the \execbrokerstate{} changes
are primarily driven by user interactions.

\begin{itemize}
    \item \codeword{OFFERED}    The \workerjob{} is being offered.
    \item \codeword{ACCEPTED}   The \workerjob{} has been accepted.
    \item \codeword{REJECTED}   The \workerjob{} offer has been rejected.
    \item \codeword{EXPIRED}    The \workerjob{} offer has expired.
\end{itemize}

When the \execbrokerservice{} replies with an \execofferset{} containing
a list of \execoffer{}s, the user can choose to do nothing, to accept one of the
\execoffer{s} from the list that best
fits their requirements, or they can reject the \execoffer{s}.

Each of the \execoffer{s} in the \execofferset{} represent a temporary reservation
for the resources listed in the \execoffer{}.
This means these resources will not available to other users while the \execoffer{s}
are still valid.

If the user does nothing, then \execbrokerstate{} of each of the \execoffer{s} will
automatically be updated to \codeword{EXPIRED}, and their associated resources will
be released, when their expiry time is reached.

If the user accepts one of the \execoffer{s} in the \execofferset{} by updating
the \execbrokerstate{} to \codeword{ACCEPTED}, the \execbrokerservice{} SHOULD
update the \execbrokerstate{} of the other \execoffer{s} in the \execofferset{} to
\codeword{REJECTED} and release the associated resources.

TODO [accept/reject/expire state-transition diagram ]

\subsubsection{Session execution}
\label{subsub-session-execution}

Once the user accepts one of the \execoffer{s}, the \execbrokerservice{} takes over and
drives the subsequent \execbrokerstate{} changes based on the timing schedule and
the results of the internal steps within the \execbrokersession{}.

\begin{itemize}
    \item \codeword{WAITING}    The \workerjob{} is waiting to activate.
    \item \codeword{PREPARING}  The resources are being prepared.
    \item \codeword{READY}      The \workerjob{} is ready to execute.
    \item \codeword{RUNNING}    The \workerjob{} is executing.
    \item \codeword{RELEASING}  The resources are being released.
    \item \codeword{COMPLETED}  The \workerjob{} has completed.
    \item \codeword{CANCELLED}  The \workerjob{} has been cancelled.
    \item \codeword{FAILED}     The \workerjob{} has failed.
\end{itemize}






It is up to the \execbrokerservice{} to select the right time to change the \workerjob{}
\codeword{phase} from \codeword{WAITING} to \codeword{PEPARING} and begin preparing the resources so that
the \workerjob{} is \codeword{READY} in time for the \codeword{starttime} declared
in the \execoffer{}.

If it will take 2 hours to transfer the data resources
from archive storage to live storage co-located with the compute resources,
then the \execworkerclass{} needs to start the \codeword{PREPARING} phase at least 2 hours
before the \codeword{starttime} declared in the \execoffer{}.

Once all the resources are ready, the \execworkerclass{} changes the \workerjob{}
\codeword{phase} to \codeword{READY} to indicate that the all the resources
are ready and the \workerjob{} is waiting to start.

The \execworkerclass{} will then wait until the \codeword{starttime} declared in the \execoffer{}
at which point it will start executing the \workerjob{} and change the \workerjob{} \codeword{phase}
to \codeword{RUNNING}.

When the \workerjob{} finishes executing, because the \dockercontainer{} finished executing,
the user closed their \jupyternotebook, or the \codeword{maxduration} was reached,
the \execworkerclass{} will change the \workerjob{} \codeword{phase} to \codeword{TEARDOWN} and
begin the process of releasing the resources.

If the \workerjob{} includes some persistent storage that should last beyond the end of the \workerjob{},
then part of the \teardown{} process may involve transferring results from the \workerjob{}
onto the persistent storage before the local storage is released.

When the \teardown{} process completes, the \workerjob{} \codeword{phase} is changed to \codeword{COMPLETED}.

If an error occurs at any time in the process, the \workerjob{} \codeword{phase} is changed to \codeword{FAILED}.
This includes any errors that occur during the \teardown{} process; for example, because
the \execworkerclass{} was unable to transfer the results onto persistent storage.
Then the \workerjob{} \codeword{phase} is changed to \codeword{FAILED}, even if the main part of the
execution completed successfully.
This is because any workflow steps that follow after this step, will depend not only on the execution being
completed, but they also need the \teardown{} data transfers to complete so that the results from this step
are in the right place for the next step to be able to access them.


\subsubsection{Update options}
\label{subsub-update-options}

Each \execsession{} has a unique URL that the client can use to monitor and update
its state.

The \execbrokerservice{} response for an \execsession{} includes a list of options
that the user may use to update or modify the \execsession{}.
Which options are available will depend on the current \codeword{state} of the
\execsession{} and the identity and permissions of the authenticated user.

If the \execsession{} is still being offered, then the list of available options
allow the user to accept or reject the offer by updating the \codeword{state} of
the \execsession{} to \codeword{ACCEPTED} or \codeword{REJECTED}.

\begin{lstlisting}[]
uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face"
href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face"
state: 'OFFERED'
expires: "2023-09-18T07:05:21"
....
....
options:
  - type: "uri:enum-value-option"
    path: "state"
    values:
    - "ACCEPTED"
    - "REJECTED"
\end{lstlisting}

Once the \execoffer{} has been accepted and the \execsession{} has started to
execute, then the list of available options will only allow the user to cancel
the execution.

\begin{lstlisting}[]
uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face"
href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face"
state: 'ACCEPTED'
expires: "2023-09-18T07:05:21"
....
....
options:
  - type: "uri:enum-value-option"
    path: "state"
    values:
    - "CANCELLED"
\end{lstlisting}

Using a dynamic list of options in this way enables the \execbrokerservice{}
to communicate to the client what actions the user is able to take
over the lifetime of an \execsession{}.

\section{The data model}
\label{sect-data-model}

As outlined in section \ref{subsub-openapi-specification}, this section of the document is
intended to compliment the \openapi{} specificaton published alongside this document.

The \openapi{} specification defines the technical details of the
\executionbroker{} \datamodel{}.
This document uses use cases and examples to describe the reasoning
behind some of the design choices.

The two documents are intended to be read together.
The machine-readable \openapi{} spcification defining the \textit{what},
and the human-readable text document describing the \textit{why}.

\subsection{The \executable{}}
\label{sub-executable}

At the simplest level the client just needs to check whether a platform is able to execute a particular
type of \excutabletask{}.
For example, \textit{"Is this platform able to run a \jupyternotebook{}?"}

In order to do this, the request needs to specify the task type, e.g. \jupyternotebook{},
along with details about it, e.g. where to fetch the notebook from.

The information in this part of the \datamodel{} will be different for each type of \executable{}.
Rather than try to model every possible type of \executable{} in one large \datamodel{},
the \datamodel{} for each type is described in an extension to the core \datamodel{}.

The \datamodel{} uses a common pattern for polymorphic
\footurl{https://swagger.io/docs/specification/v3_0/data-models/inheritance-and-polymorphism/\#polymorphism}
types based on a discriminator
\footurl{https://swagger.io/docs/specification/v3_0/data-models/inheritance-and-polymorphism/\#discriminator}
value to indicate the type of thing it is describing, followed by the specific
details for that type.

This is implemented in the \openapi{} specification as an abstract base class
containing common fields like a name and \uuid{} identifier, followed by a list
of derived types and their type identifiers.

\begin{lstlisting}[]
[+# OpenAPI schema+]
AbstractExecutable:
  type: object
  discriminator:
    propertyName: type
    mapping:
      "uri:java-program-1.0":       'JavaProgram'
      "uri:python-program-1.0":     'PythonProgram'
      "uri:docker-container-1.0":   'DockerContainer'
      "uri:singular-container-1.0": 'SingularContainer'
      "uri:jupyter-notebook-1.0":   'JupyterNotebook'
      ....
  properties:
    name:
      description: >
        A human readable name, assigned by the client.
      type: string
    uuid:
      description: >
        A machine readable UUID, assigned by the server.
      type: string
      format: uuid
    type:
      description: >
        The type discriminator.
      type: string
      format: uri
\end{lstlisting}

The derived types extend this abstract base class to include the details needed to
describe their particular type of \executablething{}.

This pattern of using a \codeword{type} discriminator to identify the type of thing, followed
by the type specific details is used in several places in the \executionbroker{} \datamodel{}.
This enables us to keep the core \datamodel{} relatively small, defining the common aspects
needed to describe an \executablething{} and the resources it needs while allowing the
\datamodel{} to be extended to describe a wide range of different types of things.

This pattern makes it easy for projects outside the core \ivoa{} community to add new
types of \executablething{s} and resources appropriate for their science domain.

Implementations do not need to understand all of the different possible types of \executable{}.
If a service doesn’t recognize a particular type, it can simply reply \codeword{NO}.

\begin{lstlisting}[]
Request  - Can this platform execute <unkown-type> ?
Response - NO
\end{lstlisting}

\subsubsection{\dockercontainer{}}
\label{subsub-dockercontainer}

The derived type for a \dockercontainer{} includes properties
to describe where to get the \dockerimage{} from, including the repository endpoint URL,
the name and version tag of the \dockerimage{} to download, and a digest checksum for
verification.

\begin{lstlisting}[]
[+# OpenAPI schema+]
DockerContainer:
  description: |
    A Docker or OCI container.
    See https://opencontainers.org/
  type: object
  title: DockerContainer
  allOf:
    - $ref: 'AbstractExecutable'
    - type: object
      properties:
        repository:
          description: >
            The image respository URL.
          type: string
        image:
          description: >
            The image name within the repository.
          type: string
        tag:
          type: string
          description: >
            The image version tag.
        digest:
          description: >
            The image digest checksum, used for verification.
          type: string
        ....
\end{lstlisting}

This results in the following message being sent to request the execution
of a \dockercontainer{}.
\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
executable:
  [+# Common fields from AbstractExecutable+]
  name: Experiment one
  type: uri:docker-container-1.0

  [+# The type specific details for a Docker container.+]
  repository: ghcr.io
  image: ivoa/analytics/Newton-Rahpson-example
  tag: 2024.08.30
  digest: sha256:ac1b....fc70
\end{lstlisting}

\subsubsection{\jupyternotebook{}}
\label{subsub-jupyternotebook}
The \datamodel{} for a \jupyternotebook{} needs to describe where to fetch the
the notebook from.

\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
executable:
  [+# Common fields from AbstractExecutable+]
  name: Experiment two
  type: uri:jupyter-notebook-1.0

  [+# The type specific details for a Jupyter notebook.+]
  notebook:
    file: https://.../example.jpnb
\end{lstlisting}

It may also include a reference to a \codeword{requirements.txt} file that describes the
additional \python{} modules needed to run the notebook.
\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
executable:
  [+# Common fields from AbstractExecutable+]
  name: Experiment two
  type: uri:jupyter-notebook-1.0

  [+# The type specific details for a Jupyter notebook.+]
  notebook:
    file: https://.../example.jpnb
  requirements:
    file: https://.../requirements.txt
\end{lstlisting}

\subsection{Resources}
\label{sub-resources}

At the next level the client may want to check whether a platform has sufficient compute resources
to execute a particular task.
For example, \textit{"Can this platform provide enough resources to run this \jupyternotebook{}?"}

In order to do this the request would not only need to describe the \executable{} itself,
but also the minimum level of compute resources needed in terms of \cpu{} cores, memory,
and the local storage space needed to execute it.

\subsubsection{Compute resources}
\label{subsub-compute-resources}

The \datamodel{} for describing compute resources is, in most cases, common to all types of \executable{},
so the \datamodel{} for these requirements are defined as part of the core \executionbroker{} \datamodel{}.

It is important to note that all of the resource requirements are optional.
A request to execute a simple \jupyternotebook{} does not need to include any resource details.

\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
executable:
  type: uri:jupyter-notebook-1.0
  notebook:
    file: https://.../example.jpnb
\end{lstlisting}

As long as the \jupyternotebook{} only needs a minimal set of resources to run,
then this is all you need to request an \execsession{} to execute it.

However, if the \jupyternotebook{} needs a minimum amount of computing resources
to function properly, then the request may include details of a compute resource
with the required minimum values.

\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
executable:
  type: uri:jupyter-notebook-1.0
  notebook:
    file: https://.../example.jpnb

resources:
  compute:
    - type: uri:generic-compute
  [*    cores:                    *]
  [*      requested:              *]
  [*        min: 4                *]
  [*    memory:                   *]
  [*      requested:              *]
  [*        min: 16               *]
\end{lstlisting}

When the \execbrokerservice{} reponds with a set of offers, it may offer more
resources than the minimum specified in the original request.
The most common scenario for this is where something like an \openstack{}
\execplatform{} can only allocate compute resources from a fixed set of templates
or `flavors` that provide specific amounts of \cpu{} cores, memory, and local storage.
In which case, the \execbrokerservice{} will select the closest match that meets
the requested criteria.
To support this, the \datamodel{} includes two sets of values, one for the request
and one for the response. This enables a client to clearly display the differences
between the request and the offer to the user.

\begin{lstlisting}[]
[+# ExecutionBroker offerset response.+]
executable:
  type: uri:jupyter-notebook-1.0
  notebook:
    file: https://.../example.jpnb
resources:
  compute:
   - type: uri:generic-compute
      cores:
  [*      requested: *]
  [*        min: 4   *]
  [*      offered:   *]
  [*        min: 8   *]
      memory:
  [*      requested: *]
  [*        min: 16  *]
  [*      offered:   *]
  [*        min: 32  *]
\end{lstlisting}

\subsubsection{Storage resources}
\label{subsub-storage-resources}

The resources section of the \datamodel{}can also be used to specify a range
of different storage resources.

There are two main types of storage resources:
\begin{itemize}
    \item External storage resources that are provided by an external source.
    \item Internal storage resources that are provided by the platform.
\end{itemize}

Within the internal storage group there are two types of resources:
\begin{itemize}
    \item Ephemeral storage available for the duration of the \workerjob{}.
    Created when the \workerjob{} starts and released when the \workerjob{} is completed.
    \item Persistent storage that exists beyond the lifetime of the \workerjob{}.
    Created before the \workerjob{} starts and remaining after the \workerjob{} has completed.
\end{itemize}

Within the persistent storage group there are another two levels of resources:
\begin{itemize}
    \item Managed resources that are created and deleted by the platform.
    \item Unmanaged resources that are created and deleted by an external entity.
\end{itemize}

The simplest of these are ephemeral storage resources managed by the execution platform.
For example, if the \jupyternotebook{} in our example requires 1024GiB of space to
perform its calculations, then this can be specified in the request by defining a
simple ephemeral storage resource.

\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
request:
  [+# Details of the executable.+]
  executable:
    ....
  [+# Details of the resources needed.+]
  resources:
    ....
    storage:
    [*  - name: scratch-space          *]
    [*    type: uri:ephemeral-storage  *]
    [*    size:                        *]
    [*      requested:                 *]
    [*        min: 1024                *]
\end{lstlisting}

To enable the \jupyternotebook{} to access this storage, we need to add a
corresponding \codeword{volume} element to the compute resource that describes
where to mount the storage resource.

The following request asks for 1024GiB of ephemeral storage
that is mounted at \codeword{/temp} in the filesystem of the compute resource.
The compute volume is linked to the storage resource by the name of the
storage resource.

\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
request:
  [+# Details of the executable.+]
  executable:
    ....
  [+# Details of the resources needed.+]
  resources:

    compute:
      - name: compute-001
        ....
        volumes:
          - name: temp-volume
            [*resource: scratch-space*]  ----\
            path: /temp                  |
            mode: rw                     |
                                         |
    storage:                             |
      - [*name: scratch-space*]         <----/
        type: uri:ephemeral-storage
        size:
          requested:
            min: 1024
\end{lstlisting}

This pattern of separating the details of how a storage resource is implemented
from the details of how it is mounted inside a computing resource is based on a
pattern used by \kubernetes{} to describe storage volumes and their mount points
within containers\footurl{https://kubernetes.io/docs/concepts/storage/volumes/}.

This pattern can also be used to define a storage resource that imports data from
an external source.
For example, if the user wanted to use the \jupyternotebook{} to analyse data stored
in an external S3 system, this can be done by defining an external storage resource
that describes where the data is,
and a corresponding volume mount inside the compute resource.

\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
request:
  [+# Details of the executable.+]
  executable:
    ....
  [+# Details of the resources needed.+]
  resources:

    compute:
      - name: compute-001
        ....
        volumes:
          - name: data-volume
            resource: source-data  ----\
            path: /data                |
            mode: r                    |
                                       |
    storage:                           |
    - name: source-data           <----/
      type: uri:amazon-s3-storage
      endpoint: https://.../echo
      bucket: example-data
\end{lstlisting}

Note that this request is defining the result the user wants to see;
namely the contents of the remote S3 bucket accessible as read-only files
in the \jupyternotebook{'s} filesystem.
It is not defining how the underlying \execplatform{} should go about making
it happen.

The \execplatform{} could use something like
s3fs-fuse\footurl{https://github.com/s3fs-fuse/s3fs-fuse} to mount the remote
S3 bucket as a directory in the \jupyternotebook{'s} filesystem,
or it could transfer a copy of the data from the S3 bucket in to local storage.
All we are defining here is the result we want to see.

This pattern of separating how data is stored outside the system
and how it appears inside a compute resource borrows heavily from the
pattern used by \kubernetes{} to describe persistent
volumes\footurl{https://kubernetes.io/docs/concepts/storage/persistent-volumes/}.

The same pattern can be used to describe a storage resource that can be used
to save the analysis results, by defining a persistent storage resource
allocated by the system, and a corresponding volume mount inside the compute resource.

\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
request:
  [+# Details of the executable.+]
  executable:
    ....
  [+# Details of the resources needed.+]
  resources:

    compute:
      - name: compute-001
        ....
        volumes:
          - name: results-volume
            resource: results-storage ----\
            path: /results                |
            mode: rw                      |
                                          |
    storage:                              |
    - name: results-storage          <----/
      type: [*uri:persistent-storage*]
      [*lifecycle: managed*]
      lifetime:
        requested:
          min: P1D
      size:
        requested:
          min: 10
          units: TiB
\end{lstlisting}

By setting the storage resource type to \codeword{uri:persistent-storage},
and setting the \codeword{lifecycle} to \codeword{managed},
the client is asking the \execbrokerservice{} to take care of allocating
the storage and managing its lifecycle.
It is up to the \execbrokerservice{} to decide where to store the data and
how make it accessible after the \workerjob{} has completed.

For example, a \scienceplatform{} may have a \rucio{} storage system co-located
with the \execplatform{} which it uses to store user generated data.
In which case the \execbrokerservice{} would respond with an \execoffer{} that
stores the results in the \rucio{} system and provides details of how the user
can access it after the \workerjob{} has completed.

\begin{lstlisting}[]
[+# ExecutionBroker offerset request.+]
request:
  [+# Details of the executable.+]
  executable:
    ....
  [+# Details of the resources needed.+]
  resources:

    compute:
      - name: compute-001
        ....
        volumes:
          - name: results-volume
            resource: results-storage ----\
            path: /results                |
            mode: rw                      |
                                          |
    storage:                              |
      - name: results-storage        <----/
        type: [*uri:rucio-storage*]
        [*lifecycle: managed*]
        lifetime:
          requested:
            min: P1D
          offered:
            min: P5D
            max: P5D
        size:
          requested:
            min: 22
            max: 22
            units: TiB
        endpoint: http://....
        domain: Project 51
        objectid: cdc78e7d-8032-497e-9a5b-01c720ea2223
\end{lstlisting}

% ZRQ - here

In this example the client requested at least 10TiB of storage available for 1 day
and in response the service is offering up to 22TiB available for 5 days stored in a
\rucio{} system co-located close to the compute platform.
It is up to the client to check if it can access the particular \rucio{} system
described in the response before it accepts the \execoffer{}.

Offering an \execoffer{} with the \codeword{lifecycle} set to \codeword{managed} and the
\codeword{lifetime.max} set to \codeword{P5D}
means that the \execbrokerservice{} is offering to manage the lifecycle of the storage.
The storage will be available until 5 days after the \workerjob{} completes and then it
will be deleted automatically.
The client doesn't need to worry about tidying up afterwards.

It is important to note that at this point in time the storage is proposed, but not yet allocated.
The persistent storage is only allocated if the client accepts this particular \execoffer{}.
This allows an \execbrokerservice{} to make multiple \execoffer{s} with different storage options,
allowing the client to select and accept the one that best fits its use case.

The same \datamodel{} can be used the other way around as well.
If the client already knows where it wants the data to be stored, for example at a specific
\vospace{} location, then it can specify this in the request.




\begin{lstlisting}[]
# ExecutionBroker client request.
request:
  # Details of the executable.
  executable:
    ....
  # Details of the resources needed.
  resources:
    ....
    compute:
    - name: "compute-001"
      ....
      volumes:
      - name: "results"
        path: "/results"
        mode: "rw"
    storage:
    - name: "results"
      type: "https://www.purl.org/ivoa.net/storage-types/vospace-storage"
      spec:
        endpoint: "http://...."
        path: "/experiment-21/results"
        lifecycle: "unmanaged"
\end{lstlisting}

It is up to the \execbrokerservice{} to work out if it is able to access the
\vospace{} location and mount it as a volume in the compute resource,
using either its own authentication, or a delegated form of the authentication
provided by the client.

If it can access the \vospace{} location, then the \execbrokerclass{} MAY respond
with an \execoffer{}, otherwise if it can't access the \vospace{} location for whatever
reason the \execbrokerclass{} MUST respond with \codeword{NO}.

Note that in this example, the client has specified the lifecycle as \codeword{unmanaged},
which means that the \execbrokerclass{} is not involved in managing the creation or deletion
of the data in \vospace.
It is also possible for the client to ask the \execbrokerservice{} to manage
data in an external resource.

\begin{lstlisting}[]
# ExecutionBroker client request.
request:
  # Details of the executable.
  executable:
    ....
  # Details of the resources needed.
  resources:
    ....
    storage:
    - name: "results"
      type: "https://www.purl.org/ivoa.net/storage-types/vospace-storage"
      spec:
        endpoint: "http://...."
        type: "container"
        path: "/experiment-21/results"
        lifecycle: "managed"
        lifetime:
          max: "P2D"
\end{lstlisting}

In this example, the client is specifying an external \vospace{} location to store the data,
but it is asking the \execbrokerservice{} to manage the lifecycle, creating the location
in \vospace{} at the start of the \workerjob{} and deleting it 2 days after the \workerjob{} completes.

This negotiation of who is responsible for creating and deleting storage resources
enables a client to put together a workflow of interconnected steps, with the
\execbrokerservice{s}
managing the lifecycle of the resources and releasing them automatically after they
are no longer needed.

\subsection{Date and time}
\label{sub-date-time}

The \codeword{datetime} part of the \datamodel{} enables the client and server to have a
conversation about when a \workerjob{} can be executed.

The client can specify one or more time periods when it would like to start the \workerjob{},
and the minimum duration that it thinks would be needed to complete the \workerjob{}.

The \execbrokerservice{} may respond with one or more \execoffer{s} that specify when the \workerjob{}
would start and the maximum duration that the \workerjob{} would be allowed to consume.
It is then up to the client to select which of the \execoffer{s} best suits its use case.

The start time for a \workerjob{} is expressed as an array of time intervals, as defined by
ISO 8601 \citep{std:iso8601}.
Specifically, type 1 or type 2 intervals (start/end and start/duration), excluding type 3 and 4 intervals
(duration/end and duration only) and excluding
repeats\footurl{https://www.iso.org/iso-8601-date-and-time-format.html}\footurl{https://en.wikipedia.org/wiki/ISO_8601}.

Note that the duration part of the interval applies to the start time, specifying a
time range during which the \workerjob{} may start.

If no duration is specified, this means an absolute start time;
e.g. the \workerjob{} SHOULD start at 11:30 on 14 August.
\begin{lstlisting}[]
# ExecutionBroker client request.
....
datetime:
- start: "2023-08-14T11:30Z"
\end{lstlisting}

If the start and end are specified, this means the \workerjob{} SHOULD start somewhere between
the start and end values;
e.g. the \workerjob{} SHOULD start between 11:30 and 12:00 on 14 August.
\begin{lstlisting}[]
# ExecutionBroker client request.
....
datetime:
- start: "2023-08-14T11:30Z/T12:00Z"
\end{lstlisting}

If a duration is specified, this means the \workerjob{} SHOULD start somewhere between
start and the start plus duration;
e.g. the \workerjob{} SHOULD start between 11:30 and 12:00 on 14 August.
\begin{lstlisting}[]
# ExecutionBroker client request.
....
datetime:
- start: "2023-08-14T11:30Z/PT30M"
\end{lstlisting}

The \execbrokerservice{} SHOULD respond with one or more \execoffer{s} that start within
the ranges specified in the request.
The start times in the \execoffer{s} MAY be more precise than the start times in the request,
but they MUST all occur within at least one of the ranges specified in the request.

The client MAY also specify the maximum and minimum execution duration,
expressed as time periods as defined by ISO 8601.

For example, for the unattended batch mode execution of an \dockercontainer, the user might not be concerned about
when their \workerjob{} starts, but they may want to specify the minimum duration needed to complete the task.

In which case, the client may simply request a minimum duration of 1 hour.
\begin{lstlisting}[]
# ExecutionBroker client request.
....
datetime:
- duration:
    min: "P1H"
\end{lstlisting}

The \execbrokerservice{} MAY respond with \execoffer{s} that start at different times and
set different values for the maximum duration.

It MAY offer a maximum duration of 1 hour in the morning, starting at 11:00.
\begin{lstlisting}[]
# ExecutionBroker server response.
....
datetime:
- start: "2023-08-14T11:30Z"
  duration:
    min: "P1H"
    max: "P1H"
\end{lstlisting}

It MAY also offer a longer allocation of 4 hours later in the evening,
starting sometime between 22:00 and 23:00.
\begin{lstlisting}[]
# ExecutionBroker server response.
....
datetime:
- start: "2023-08-14T22:00Z/PT1H"
  duration:
    min: "P1H"
    max: "P4H"
\end{lstlisting}

Different values for start time and duration can be combined with different values for the
compute resources to make a range of different \execoffer{s} to the client.

For example, if a client asks for 2 cores and 2G of memory for 1 hour sometime on 14 August:
\begin{lstlisting}[]
# ExecutionBroker client request.
  ...
resources:
  compute:
    cores:
      min: 2
    memory:
      min: 2
datetime:
  - start: "2023-08-14/P1D"
    duration:
      min: "P1H"
\end{lstlisting}

The  \execbrokerservice{} may respond with 2 \execoffer{s},
the minimum 2 cores and 2G of memory for 1 hour starting at 11:30,
and a larger \execoffer{} of up to 8 cores and 8Gb of memory for up to 4 hours
starting between 22:00 and 23:00.

\begin{lstlisting}[]
# ExecutionBroker server response.
offers:
- name: "offer-001"
  executable:
    ...
  resources:
    compute:
      cores:
        min: 2
        max: 2
      memory:
        min: 2
        max: 2
  datetime:
    - start: "2023-08-14T11:30Z"
      duration:
        min: "P1H"
        max: "P1H"

- name: "offer-002"
  executable:
    ...
  resources:
    compute:
      cores:
        min: 2
        max: 8
      memory:
        min: 2
        max: 8
  datetime:
    - start: "2023-08-14T22:00Z/T23:00Z"
      duration:
        min: "P1H"
        max: "P4H"
\end{lstlisting}

It is then up to the client to decide which \execoffer{} better suits their use case.
Accept the limited \execoffer{} in the morning, or accept the more generous \execoffer{} with
more resources and more time later in the day.

If an \execbrokerservice{} is offering more than one option for the \codeword{datetime}
section it MUST make separate \execoffer{s} for each different option.
Even if all of the other parameters are the same, e.g. compute and storage resources, the
\execbrokerservice{} MUST NOT include more than one time slot in the same \execoffer{}.
Technically the \datamodel{} allows an array of values for the \codeword{datetime} section,
but this would impose unnecessary complexity on the client for no real gain in user experience.




\subsection{Metadata roles}
\label{sub-metadata-roles}

The full description of an \executablething{} will include several layers of metadata
provided by different actors playing different roles within the publishing process.

For our square root example we can identify a number of roles that would each provide
layers of the picture nedeed to fully describe an \executablething{}.

The players:
\begin{itemize}
    \item The developer  - The person who wrote the \pythonprogram{}
    \item The packager   - The person who packaged it in a \dockercontainer{}
    \item The publisher  - The person who published it in a discovery service
    \item The user agent - The person who wants to use the software
\end{itemize}

\subsubsection{The developer}
\label{subsub-software-developer}

The first layer of metadata comes from the person who wrote the software.
They have detailed knowledge of what the software does, what execution environment it needs,
and what the inputs and outputs are.

For our square root example, it is a \pythonprogram{} which needs a platform with the \python{} runtime installed,
and a list of the \python{} libraries that the program relies on.

\begin{lstlisting}[]
executable:
  name: Newton-Raphson example
  type: uri:python-program-1.0
  requirements:
  - numpi: ""
  - astropy: ">= 6.1"
\end{lstlisting}

The developer also understands how much memory their program needs, whether it can make use of multiple cpu cores,
and whether it can make use of a \gpu{} accelerator.

\begin{lstlisting}[]
resources:
  compute:
    - type: uri:generic-compute
      cores:
        requested:
          min: 4
      memory:
        requested:
          min: 16
      ....
\end{lstlisting}

The developer also knows about what inputs and outputs the program expects and what file
formats can it can handle.
% needs work
%https://github.com/ivoa-std/ExecutionBroker/issues/89

\begin{lstlisting}[]
executable:
  name: Newton-Raphson example
  type: uri:python-program-1.0
  ....
  parameters:
    - name: INPUT_FILE
      type: uri:filesystem-file
      mode: readonly
      description:
        A table containing a list of numbers to be processed,
        formatted as comma separated text (CSV) or an IVOA
        VOTable.
      formats:
        - type: uri:filetype-text-csv
          ....
        - type: uri:filetype-ivoa-votable
          ....
    - name: INPUT_COLUMN
      type: uri:environment-variable
      description:
        The column name within the 'INPUT_FILE' to use.
\end{lstlisting}

\subsubsection{The packager}
\label{subsub-software-packager}

Although it is possible to publish our square root example as a stand alone \pythonprogram{},
it is not easy to describe the installation process in sufficient detail for it to be
automatically deployed on a range of different platforms.

A more portable solution would be to package the \pythonprogram{} in a \dockercontainer{},
installing and configuring the software along with all of its dependencies inside the container.

This is often done as part of the software development process, but it is a separate
step that could be implemented by a different person.
To make this distinction clear we can refer to this person or role as \textit{'the packager'}.

This step packages the \pythonprogram{} along with any \python{} modules it requires,
the \pythonruntime{}, and any operating system components it requires, into a single
standard format binary file, making it much easier to deploy.

The packager would then create a new \metadoc{} that describes the \executablething{}
as a \dockercontainer{} rather than a \pythonprogram{}, with details of where to
find the \dockerimage{}, including the image name, the version tag and an optional
checksum.

\begin{lstlisting}[]
[+# Executable metadata+]
executable:
  name: Newton-Raphson example
  [*type: uri:docker-container-1.0*]
  [*repository: ghcr.io*]
  [*image: ivoa/analytics/Newton-Raphson-albert*]
  [*tag: 2024.08.30*]
  [*digest: sha256:ac1b........fc70*]
  ....
\end{lstlisting}

Depending on how the software is packaged in the container the
new \metadoc{} may update the description of the inputs and outputs,
and link them to specific locations in the filesystem of a compute
resource.
% needs work
%https://github.com/ivoa-std/ExecutionBroker/issues/89

\begin{lstlisting}[]
[+# Executable metadata+]
executable:
  name: Newton-Raphson example
  type: uri:docker-container-1.0
  ....
  parameters:
    - [*name: INPUT_FILE*]            <----\
      type: uri:filesystem-file        |
      format:                          |
        - type: uri:ivoa-votable       |
          ....                         |
                                       |
resources:                             |
  compute:                             |
    - type: uri:generic-compute        |
      volumes:                         |
        - type: uri:filesystem-mount   |
          [*parameter: INPUT_FILE*]     ---/
          [*filepath: /data/input-data.vot*]
          [*mode: readonly*]
          ....
\end{lstlisting}

\subsubsection{The publisher}
\label{subsub-metadata-publisher}

This role represents the person who publishes metadata about the software in a discovery service.

Typically changes made at this level would include adding to the description
of what the software does, placing it in a particular context to aid discovery,
or modifying the execution environment to configure the softweare for a particular domain.
This might include things like:

\begin{itemize}
    \item A project specific discovery service that only includes software vetted by the project.
          Execution platforms within the project would only accept curated \metadoc{s}
          from that discovery service.
    \item A domain specific discovery service that modifies the execution environment, configuring
          the software to analyse a particular type of data.
    \item A catalog of \metadoc{s} maintained as part of a teaching course, modifying the
          execution environment to integrate the software into the local IT system and setting
          parameters to configure it to match the course notes.
\end{itemize}

\subsubsection{The end user}
\label{subsub-software-user}

The user starts with \metadoc{} from the software discovery service and adds
some information describing how they want to use the software.

This may include selecting the data resources that they want to use
and adding them to the \metadoc{} as data resources.
This means the \execbrokerservice{}
can include the time needed to transfer the data to local storage before the
\execsession{} is started.

\begin{lstlisting}[]
executable:
  ....
resources:
  ....
  data:
    - [*name: input data*]
      [*type: uri:simple-data-resource*]
      [*location: http:data.example.org/....*]
      ....
\end{lstlisting}


Linking the data resources with volumes on the corresponding compute resources enables
the \execbrokerservice{} to mount the data resources at the correct location in
the compute resource's filesystem.

\begin{lstlisting}[]
resources:
  compute:
    - type: uri:generic-compute
      volumes:
        - type: uri:filesystem-mount
          parameter: INPUT_FILE
          [*resource: input-data*]          ----\
          filepath: /data/input-data.vot    |
          mode: readonly                    |
          ....                              |
  data:                                     |
    - [*name: input-data*]                  <---/
      type: uri:simple-data-resource
      location: http:data.example.org/....
      ....
\end{lstlisting}

The user can also update the compute resource requirements to reflect how they plan
to use the software.
In the case of our square root example, the compute resource requirements set by the
developers will reflect the original intent of simply demonstrating how to use the
\pythonprogram{}.

However, if the user intends to use the software to analyse a much larger \dataset{}
they can update the compute resource requirements from 4 cores and 16 GiB of memory
to 32 cores and 128 GiB of memory to match their new requirements.

\begin{lstlisting}[]
resources:
  compute:
  - type: uri:generic-compute
    cores:
      requested:
        [*min: 32*]
    memory:
      requested:
        [*min: 128*]
\end{lstlisting}

The user can use the \codeword{schedule} section of the \metadoc{}
to describe when they want the \execsession{} to start.

This is covered in more detail in section \ref{sub-date-time} of this document.
TODO - add more detail here

\pagebreak

\section{Federated architecture}
\label{sect-sect-federation}

The \execbrokerservice{} is designed to be used at multiple levels within an organization.

At the low level, an \execbrokerclass{} serice may be implemented as a
single web-application linked to a simple \docker{} execution service.
The configuration may be hard coded to only accept a fixed white list of container images
and a fixed allocation of compute resources for each job{}.
e.g. the platform will only run a fixed set of container images, and only provides 2 \cpu{} cores, 2G of memory and a maximum 1HR execution time.

A project or organization may deploy several of these low level services,
providing a range of different capabilities.

Given a new \workerjob{} to execute, a client can simply poll each of the services to see which ones are
able to execute it.

The client does not need to have any prior knowledge about the services apart from their
endpoint address.
This information could come from the \ivoa{} Registry, or it could simply
be provided as a flat list in a configuration file for the client.

A more flexible architecture could add another \execbrokerservice{}
a level above the low level task specific services.
This service would be configured with a list of the local task specific services
and act as an aggregator service sitting between the client and the low level services.
This aggregator service would forward a copy of each request it receives to each of the low level services in its list and
then aggregate the \execoffer{s} from the individual responses into a single response that is sent back to the client.

A simple aggregator service would just forward every request to all the low level services below it,
regardless of what the request contained.

A more complex aggregator service may have some prior knowledge about what types of task or compute resources
each of the lower level services are able to provide, enabling it to make more informed decisions about
which low level serviecs to send the requests to.

The aggregator service may also have an understanding of the location of \dataset{s} within the organization and
be able to route requests to different low level services depending on which \dataset{s} were required.

The aggregator service may expose the individual low level \execbrokerclass{} endpoints in its responses,
or it may implement a proxy interface hiding the individual services.

This configuration could be used to provide an \execbrokerservice{} interface
that bridges a firewall. Providing a public interface for external clients and forwarding the requests
to internal \execbrokerservice{s} that are not accessible from outside the
firewall.

A large organization with multiple sites could also deploy a single high level \execbrokerclass{}
interface that handles requests for the whole organization, forwarding individual requests to mid-level
\execbrokerservice{s} at each site which in turn forwards the requests to
individual low-level \execbrokerservice{s} for each platform within the local sites.

The implementation at each level may be different, providing different levels of routing
and aggregation based on internal knowledge of the capabilities of the level below,
but the \execbrokerclass{} interfaces would all be the same.

This could be expanded to include another level of \execbrokerservice{s} that
cross organization bondaries, providing a gateway that allows users from one project to access services
from other projects and organizations. This gateway service may also provide an authentication translation
layer between external public accounts and internal project specific identities and authentication methods.

\pagebreak

\section{Service specification}
\label{sect-service-specification}

Details of the request and response messages.
Timeline of request, offer and accept.

\subsection{General requirements}
\label{sub-general-requirements}

\begin{itemize}
    \item ....
    \item ....
    \item ....
\end{itemize}

\subsection{ExecutionBroker}
\label{sub-execution-planner-spec}

The main \execbrokerclass{} \webservice{} implements two interfaces;
a \codeword{POST} method for sending requests, and a \codeword{GET/POST}
interface for accessing \execoffer{s} and updating their status.

\subsubsection{Request method}
\label{subsub-execution-planner-request}

The main \execbrokerclass{} \webservice{} method is a \codeword{POST} \webservice{} that accepts
a \codeword{request} as defined in section \ref{sub-datamodel-request} of the \datamodel{}.

An \execbrokerclass{} \webservice{} MUST be able to accept both \yaml{} or \json{} serializations
of the \codeword{request}.
The serialization SHOULD be declared in the HTTP request using the \codeword{Content-Type} header.
\begin{itemize}
    \item \codeword{application/yaml} for \yaml{}.
    \item \codeword{application/json} for \json{}.
    \item If the serialization is not declared, the default is assumed to be \yaml{}.
    \item If the both \yaml{} and \json{} are declared, the default is assumed to be \yaml{}.
\end{itemize}

\begin{lstlisting}[]
POST /request HTTP/1.1
Host: foo.example.org
Content-Type: application/yaml

request:
  executable:
    ....
  resources:
    ....
\end{lstlisting}

\begin{lstlisting}[]
POST /request HTTP/1.1
Host: foo.example.org
Content-Type: application/json

"request": {
  "executable": {
    ....
    }
  "resources": {
    ....
    }
  }
\end{lstlisting}

The \execbrokerservice{} will evaluate the request and respond with either a positive
or negative \codeword{response} depending on whether the platform is able to execute the
task descibed in the \codeword{request}.

An \execbrokerservice{} MUST be able to provide both \yaml{} and \json{} serializations
of the \codeword{response}.
The serialization is determined by the \codeword{Accept} header in the HTTP request.
\begin{itemize}
    \item \codeword{application/yaml} for \yaml{}.
    \item \codeword{application/json} for \json.
    \item If the serialization is not declared, the default is assumed to be \yaml{}.
    \item If the both \yaml{} and \json{} are declared, the default is assumed to be \yaml{}.
\end{itemize}

If the platform IS able to execute the task, then the \webservice{} will respond with a
HTTP response code \codeword{200} and a positive \codeword{response} content as defined
in section \ref{subsub-datamodel-positive-response}.

\begin{lstlisting}[]
HTTP/1.1 200 OK
Content-Type: application/yaml; charset=utf-8

response:
  result: 'YES'
  offers:
    ....
\end{lstlisting}

If the platform IS NOT able to execute the task, then the \webservice{} will respond with a
HTTP response code \codeword{200} and a negative \codeword{response} content as defined
in section \ref{subsub-datamodel-negative-response}.

\begin{lstlisting}[]
HTTP/1.1 200 OK
Content-Type: application/yaml; charset=utf-8

response:
  result: 'NO'
  reasons:
    ....
\end{lstlisting}

The \codeword{request} method is NOT idempotent. Sending the same \codeword{POST} request
twice MAY return different results.

\begin{itemize}
    \item The list of \execoffer{s} in a response depends on the resources available at the point when the request is made.
    \item Sending a request generates a new list of \execoffer{s}.
    Those \execoffer{s} will contain references to resources that are reserved for the lifetime of the \codeword{offer}.
    \item If a second request is made soon after the first request,
    then the \execoffer{s} made in response to the first request may still be active.
    In which case, the resources listed in those \execoffer{s} may still be reserved,
    reducing the resources available for the second request.
    \item If a second request is made some time after the first, then
    other users may have requested and accepted \execoffer{s} reducing the resources available for the second request,
    or the platform may have completed some tasks, increasing the resources available for the second request.
\end{itemize}

\subsubsection{Offer GET}
\label{subsub-execution-planner-offer-get}

The \codeword{offer} \codeword{GET} method requests the current state of an \execoffer{}.
The identifier for the \execoffer{} is encoded in the URL of the HTTP request by appending the
identifier to the path of the HTP request URL.

\begin{lstlisting}[]
http[s]://foo.example.org/offers/{identifier}
\end{lstlisting}

The \datamodel{} for the content of the response is described in section
\ref{subsub-datamodel-offer}.

An \execbrokerservice{} MUST be able to provide both \yaml{} and \json{} serializations
of the \codeword{offer}.
The serialization is determined by the \codeword{Accept} header in the HTTP request.
\begin{itemize}
    \item \codeword{application/yaml} for \yaml{}.
    \item \codeword{application/json} for \json{}.
    \item If the serialization is not declared, the default is assumed to be \yaml{}.
    \item If the both \yaml{} and \json{} are declared, the default is assumed to be \yaml{}.
\end{itemize}

\begin{lstlisting}[]
GET /offers/2c89f536-3fff-48f7-943f-bcc5c3225be7 HTTP/1.1
Host: foo.example.org
Accept: application/yaml
\end{lstlisting}

\begin{lstlisting}[]
HTTP/1.1 200 OK
Content-Type: application/yaml; charset=utf-8

offer:
  ident: "2c89f536-3fff-48f7-943f-bcc5c3225be7"
  status: 'OFFERED'
  expires: "2023-09-18T07:05:21"
  request:
    executable:
      ....
    resources:
      ....
\end{lstlisting}

\subsubsection{Offer POST}
\label{subsub-execution-planner-offer-post}

The \codeword{offer} \codeword{POST} method provides the client with a method for
updating the status of an \codeword{offer}.

The identifier for the \execoffer{} is encoded in the URL of the HTTP request by appending the
identifier to the path of the HTP request URL.

\begin{lstlisting}[]
http[s]://foo.example.org/offers/{identifier}
\end{lstlisting}

The content of the \codeword{POST} method should contain the properties
of the \codeword{offer} that the client wants to change, as defined by
the \codeword{offer} \datamodel{} in section \ref{}.
\begin{itemize}
    \item Currently, only the \codeword{status} of an \codeword{offer} is modifiable.
    \item The \codeword{POST} data MAY contain the \codeword{offer} \codeword{ident}.
    \begin{itemize}
        \item If the \codeword{offer} \codeword{ident} is presnet it MUST match the identifier used in the URL.
    \end{itemize}
\end{itemize}

An \execbrokerclass{} \webservice{} MUST be able to accept both \yaml{} or \json{} serializations
of the \codeword{offer}.
The serialization SHOULD be declared in the HTTP request using the \codeword{Content-Type} header.
\begin{itemize}
    \item \codeword{application/yaml} for \yaml{}.
    \item \codeword{application/json} for \json{}.
    \item If the serialization is not declared, the default is assumed to be \yaml{}.
    \item If the both \yaml{} and \json{} are declared, the default is assumed to be \yaml{}.
\end{itemize}

\begin{lstlisting}[]
POST /offers/2c89f536-3fff-48f7-943f-bcc5c3225be7 HTTP/1.1
Host: foo.example.org
Content-Type: application/yaml

offer:
  ident: "2c89f536-3fff-48f7-943f-bcc5c3225be7"
  status: 'ACCEPTED'
\end{lstlisting}

If the update is successful, the content of the response is the updated state of the \codeword{offer}.
The serialization of the response is determined by the \codeword{Accept} header in the HTTP request
using the same rules as the \codeword{GET} method.

\begin{lstlisting}[]
HTTP/1.1 200 OK
Content-Type: application/yaml; charset=utf-8

offer:
  ident: "2c89f536-3fff-48f7-943f-bcc5c3225be7"
  status: 'ACCEPTED'
  expires: "2023-09-18T07:05:21"
  request:
    executable:
      ....
    resources:
      ....
\end{lstlisting}

If the update is not successful, the content of the response ... TBD ...


\pagebreak

\section{The \datamodel{}}
\label{sect-datamodel}

The following section describes the core \datamodel{} using a YAML syntax
to define the structure with comments to describe the elements.

Note that this does not tie the \execbrokerclass{} specification to a YAML representation.
The intention is to define a portable \datamodel{} that can be serialised
into a number of different formats, including YAML, JSON and XML.

\subsection{Type URIs and specifications}
\label{sub-type-and-spec}

Several parts of the \ivoa{} \executionbroker{} \datamodel{} follow a common pattern, using a URI
to identify a \codeword{type} followed a \codeword{spec} section to fill in the details.
This pattern is adopted from a similar pattern used by \kubernetes{}.

\begin{lstlisting}[]
# A URI to identify the type of thing.
type: "uri:type-a"

# Type specific details about the thing.
spec:
  name: "...."
  size: "...."
  ....
\end{lstlisting}

However, the \executionbroker{} \datamodel{} specifically recomends using a long lasting resolvable
HTTP URL as the type identifier.

\begin{lstlisting}[]
# A URL to identify the type of thing.
type: "http://www.example.com/type-a"

# Type specific details about the thing.
spec:
  name: "...."
  size: "...."
  ....
\end{lstlisting}

The reasoning behind this is that if a service provider notices several clients are requesting
\textit{'encabulation'} on their compute nodes, the service provider may not know what this means.
However, if we use a resolvable URL to identify the type, a service provider can use it to find a
human readable description\footurl{https://tinyurl.com/encabulation} and decide whether they want
to provide it on their platform.

To the \webservice{} implementation, both of these requests are just a simple string comparison against a
dictionary of known terms:

\begin{lstlisting}[]
Request  - Can you provide 'encabulation' ?
Response - YES|NO
\end{lstlisting}

\begin{lstlisting}[]
Request  - Can you provide 'https://tinyurl.com/encabulation' ?
Response - YES|NO
\end{lstlisting}

The second form provides a simple mechanism for linking the type identifier to a human readable description.

While it is possible to use \ivoa{} registry URIs to identify types, this specification
recomends using a simple HTTP URLs. This lowers the barrier to entry and makes it simpler for the end user
to resolve the URL into a description.

\subsection{\execbrokerclass{} request}
\label{sub-datamodel-request}

The main section of the \execbrokerclass{} \datamodel{} describes the \codeword{request} block,
which describes an \executablething{} and the resources needed to execute it.
\\
\\
The \codeword{request} block is used in several places in the \datamodel{}:
\begin{itemize}
    \item The main body of an \execbrokerclass{} \codeword{request}.
    \item The main body of each \codeword{offer} in an \execbrokerclass{} response.
    \item The main body of an \execworkerclass{} \workerjob{} record.
\end{itemize}

\subsubsection{Executable}
\label{subsub-datamodel-executable}

The \codeword{executable} section of the \codeword{request} block describes the \executablething{}.
The core \datamodel{} simply defines this section as containing a URI identifying the \codeword{type} of executable,
and a \codeword{spec} placeholder for the type specific details.

\begin{lstlisting}[]
request:
  executable:
    # A URI identifying the executable type.
    type: !!str
    # Type specific details of the executable type.
    spec:
      ....
\end{lstlisting}

The content of the \codeword{spec} section depends on the \codeword{type} of executable.
The core \datamodel{} defines 4 types of \executable{};
\jupyter{} and \zeppelin{} notebooks, and OCI (Docker) and \singularity{} containers.
The structure is designed to make it easy for 3rd parties to extend the \datamodel{}
by defining their own types of \executable{}.

\subsubsection{Jupyter notebook}
\label{subsub-datamodel-jupyter-notebook}

The \datamodel{} extension for a \jupyternotebook{} defines the URL
to identify the \codeword{type}:
\begin{lstlisting}[]
https://www.purl.org/ivoa.net/executable-types/jupyter-notebook
\end{lstlisting}
\hfill \break
The \datamodel{} extension for a \jupyternotebook{} contains:
\begin{itemize}
    \item A URL to download the \codeword{notebook} from.
    \item An optional URL to download a \codeword{requirements} file that lists any additional
    \python{} modules needed by the notebook.
    \begin{itemize}
        \item The syntax of the \codeword{requirements} file is defined by the \codeword{pip}
        Requirements File Format\footurl{https://pip.pypa.io/en/stable/reference/requirements-file-format/}.
        \item The URL for the \codeword{requirements} file may be an absolute URL, or a relative URL based on the notebook URL.
    \end{itemize}
    \item An \codeword{endpoint} URL that points to the live notebook platform.
    \begin{itemize}
        \item The \codeword{endpoint} URL is set by the \execworkerclass{} once the \workerjob{} is \codeword{READY}.
    \end{itemize}
\end{itemize}

\begin{lstlisting}[]
request:
  executable:
    # A URL identifying the executable type.
    type: "https://www.purl.org/ivoa.net/executable-types/jupyter-notebook"
    # Type specific details of the executable type.
    spec:
      # A URL pointing to the notebook source code.
      notebook: !!str
      # A URL pointing to an optional requirements file.
      requirements: !!str
      # A URL pointing to the live notebook interface.
      endpoint: !!str
\end{lstlisting}

\subsubsection{Zeppelin notebook}
\label{subsub-datamodel-zeppelin-notebook}

The \datamodel{} extension for a \zeppelin{} notebook defines the URL
to identify the \codeword{type}:
\begin{lstlisting}[]
https://www.purl.org/ivoa.net/executable-types/zeppelin-notebook
\end{lstlisting}
\hfill \break
The \datamodel{} extension for a \zeppelin{} notebook contains:
\begin{itemize}
    \item A URL to download the \codeword{notebook} from.
    \item An optional URL to download a \codeword{requirements} file that lists any additional
    \python{} modules needed by the notebook.
    \begin{itemize}
        \item The syntax of the \codeword{requirements} file is defined by the \codeword{pip}
        Requirements File Format\footurl{https://pip.pypa.io/en/stable/reference/requirements-file-format/}.
        \item The URL for the \codeword{requirements} file may be an absolute URL, or a relative URL based on the notebook URL.
    \end{itemize}
    \item An \codeword{endpoint} URL that points to the live notebook platform.
    \begin{itemize}
        \item The \codeword{endpoint} URL is set by the \execworkerclass{} once the \workerjob{} is \codeword{READY}.
    \end{itemize}
\end{itemize}

\begin{lstlisting}[]
request:
  executable:
    # A URL identifying the executable type.
    type: "https://www.purl.org/ivoa.net/executable-types/zeppelin-notebook"
    # Type specific details of the executable type.
    spec:
      # A URL pointing to the notebook source code.
      notebook: !!str
      # A URL pointing to an optional requirements file
      requirements: !!str
      # A URL pointing to the live notebook interface.
      endpoint: !!str
\end{lstlisting}

\subsubsection{Docker container}
\label{subsub-datamodel-docker-container}

The \datamodel{} extension for an \dockercontainer{} defines the URL
to identify the \codeword{type}:
\begin{lstlisting}[]
https://www.purl.org/ivoa.net/executable-types/docker-container
\end{lstlisting}
\hfill \break
The \datamodel{} extension for an \dockercontainer{} contains:
\begin{itemize}
    \item The \codeword{repo} endpoint address of the repository to download the container image from.
    \item The name of the \codeword{image} to download.
    \item The \codeword{tag} identifier to specify which version to download.
    \item The \codeword{digest} identifier to specify which version to download.
    \item An optional list of \codeword{platform} elements which contain:
    \begin{itemize}
        \item The operating system, \codeword{os}, that the container is built for (default \codeword{linux}).
        \item The system \codeword{architecture} that the container is built for (default \codeword{amd64}).
    \end{itemize}

    % TODO how to specify the network ports.

\end{itemize}

\begin{lstlisting}[]
request:
  executable:
    # A URL identifying the executable type.
    type: "https://www.purl.org/ivoa.net/executable-types/oci-container"
    # Type specific details of the executable type.
    spec:
      # The repository to download the image from.
      repo: !!str
      # The name of the container image.
      image: !!str
      # The image tag to specify the version.
      tag: !!str
      # The image digest to specify the version.
      digest: !!str
      # Details of the platform(s) that the container is built for.
      platforms:
      - # The system architecture the container is built for.
        architecture: !!str # default: "amd64".
        # The operating system the container is built for.
        os: !!str # default: "linux".
\end{lstlisting}

\subsubsection{Singularity container}
\label{subsub-datamodel-singularity-container}

The \datamodel{} extension for a \singularity{} container defines the URL
to identify the \codeword{type}:
\begin{lstlisting}[]
https://www.purl.org/ivoa.net/executable-types/singularity-container
\end{lstlisting}
\hfill \break
The \datamodel{} extension for a \singularity{} contains:
\begin{itemize}
    \item ....
\end{itemize}

\begin{lstlisting}[]
request:
  executable:
    # A URL identifying the executable type.
    type: "https://www.purl.org/ivoa.net/executable-types/oci-container"
    # Type specific details of the executable type.
    spec:
      ....
      ....
\end{lstlisting}

\subsubsection{Resources}
\label{subsub-datamodel-resources}

The \codeword{resources} section of the \codeword{request} block describes the resources needed
to execute the task.

The \codeword{resources} section contains two lists of resources, one for
\codeword{compute} resources and one for \codeword{storage} resources.
Separating these into two lists enables many-to-many relationships between the \codeword{compute}
and \codeword{storage} resources.

\begin{lstlisting}[]
request:
  # Details of the executable
  executable:
    ....
    ....
  # Details of the resources
  resources:
    # List of compute resources
    compute:
      ....
      ....
    # List of storage resources
    storage:
      ....
      ....
\end{lstlisting}

\subsubsection{Compute resources}
\label{subsub-datamodel-compute-resources}

The \datamodel{} for a \codeword{compute} resource contains:
\begin{itemize}
    \item A \codeword{name} for the resource, unique within the \codeword{request}.
    \item An optional count of how many \codeword{replicas} of the resource are needed.
    \item A URL identifying the \codeword{type} of resource.
    \item A \codeword{spec} placeholder for the type specific details.
\end{itemize}

\begin{lstlisting}[]
request:
  resources:
    compute:
    - # A unique name for this compute resource.
      name: !!str
      # A count of how many replicas are needed.
      replicas: !!int # default: 1
      # A URI identifying the compute resource type.
      type: !!str
      # Type specific details of the compute resource.
      spec:
        ....
        ....
\end{lstlisting}

The \datamodel{} structure is designed to make it easy for 3rd parties
to extend the \datamodel{} by defining their own types of computing
resources.

\subsubsection{Generic compute resources}
\label{subsub-datamodel-generic-compute}

The core \datamodel{} defines a generic computing resource
that has cpu cores, memory and a hierarchical filesystem.
This represents a common abstraction of a virtual machine,
an OCI container runtime or a \kubernetes{} Pod.

The data model for a generic computing resource defines the URL to identify
the type:
\begin{lstlisting}[]
https://www.purl.org/ivoa.net/resource-types/generic-compute
\end{lstlisting}
\hfill \break
The \codeword{spec} for a generic computing resource may contain the following optional properties:
\begin{itemize}
    \item The minimum number of cpu cores needed, set by the client in the \execbrokerclass{} \codeword{request}.
    \item The maximum number of cpu cores available, set by the \execbrokerservice{} in an \codeword{offer}.
    \item The minimum amount of memory needed (in GiB), set by the client in the \execbrokerclass{} \codeword{request}.
    \item The maximum amount of memory available (in GiB), set by the \execbrokerservice{} in an \codeword{offer}.
\end{itemize}

\begin{lstlisting}[]
request:
  resources:
    compute:
    - # A unique name for this compute resource.
      name: !!str
      # A count of how many replicas are needed.
      replicas: !!int # default: 1
      # A URL identifying the compute resource type.
      type: "https://www.purl.org/ivoa.net/resource-types/generic-compute"
      spec:
        # The minimum and maximum number of cpu cores needed.
        cores:
          min: !!float # default: 1
          max: !!float # default: cores.min
        # The minimum and maximum amount of memory needed, in GiB.
        memory:
          min: !!float # default: 1
          max: !!float # default: memory.min
        ....
\end{lstlisting}

The \codeword{spec} for a generic computing resource may contain
a list of \codeword{volumes} that describe how a storage resource
should be mounted in the compute resource's filesystem.

\hfill \break
Each \codeword{volume} element contains:
\begin{itemize}
    \item A \codeword{name} for the volume, unique within this \codeword{request}.
    \item A reference (by name) to the corresponding storage \codeword{resource}.
    \item The \codeword{path} that the volume should be mounted at in the compute resource's filesystem.
    \item The read write access mode ('\codeword{rw}' for read and write, or '\codeword{ro}' for read only).
\end{itemize}

\begin{lstlisting}[]
request:
  resources:
    compute:
    - # A unique name for this compute resource.
      name: !!str
      # A count of how many replicas are needed.
      replicas: !!int # default: 1
      # A URL identifying the compute resource type.
      type: "https://www.purl.org/ivoa.net/resource-types/generic-compute"
      spec:
        ....
        # A list of volume mounts.
        volumes:
        - # A unique name for this volume.
          name: !!str
          # The name of the storage resource this volume is linked to.
          resource: !!str
          # The path in the compute resource's filesystem.
          path: !!str
          # The access mode, 'rw' or 'ro'.
          mode: !!str
\end{lstlisting}

\subsubsection{Storage resources}
\label{subsub-datamodel-storage-resources}

\begin{lstlisting}[]
....
....
\end{lstlisting}

\subsubsection{Authentication}
\label{subsub-datamodel-authentication}

\begin{lstlisting}[]
....
....
\end{lstlisting}

\subsubsection{DateTime}
\label{subsub-datamodel-datetime}

\begin{lstlisting}[]
....
....
\end{lstlisting}

\subsection{Response}
\label{sub-datamodel-response}

\subsubsection{Positive response}
\label{subsub-datamodel-positive-response}
A positive (yes, this platform can execute the task) \execbrokerservice{} response contains \codeword{result}
set to \codeword{YES}, and a list of \execoffer{s}.

\subsubsection{Offer}
\label{subsub-datamodel-offer}

Each \execoffer{} contains a unique identifier a status value and an expiry time for the \execoffer{}
followed by an updated copy of the \codeword{request} with details
filled in by the \execbrokerservice{}.

The \codeword{offer} \codeword{status} ....

\begin{lstlisting}[]
response:
  # The top level result.
  result: !!str 'YES'
  # A list of offers to match the request.
  offers:
  - # Unique identifier for this offer.
    ident:  !!str
    # Status of this offer.
    status: !!str # [OFFERED, ACCEPTED, REJECTED, EXPIRED]
    # Expiry date for this offer.
    expires: !!str # ISO 8601 date time
    # An updated copy of the request body.
    request:
      executable:
        ....
      resources:
        ....
      datetime:
        ....
\end{lstlisting}

\subsubsection{Negative response}
\label{subsub-datamodel-negative-response}

A negative (no, this platform cannot execute the task) \execbrokerservice{} response contains \codeword{result}
set to \codeword{NO}, followed by an optional list of reasons
explaining why the request was rejected.

Each \codeword{reason} consists of a \datamodel{} path to an element in the original
\codeword{request} and a text message explaining why the element was rejected.

\begin{lstlisting}[]
response:
  # The top level result.
  result: !!str 'NO'
  # A list of reasons explaining why the request was rejected.
  reasons:
  - # The path to an element in the request body
    path: !!str
    # A text message why the element was rejected.
    text: !!str
\end{lstlisting}

%TODO more examples of reasons ...

\subsection{Session}
\label{sub-datamodel-session}

An \execworkerclass{} \workerjob{} record consists of a unique identifier for the \workerjob{},
and a \codeword{phase} indicating what execution stage the \workerjob{} has reached.
Followed by an updated copy of the request \codeword{body} from the \execbrokerclass{}
\execoffer{} with details filled in by the \execworkerclass{} as the \workerjob{} is executed.

\begin{lstlisting}[]
job:
  # A unique identifier for this session
  ident: !!str
  # The current execution phase.
  phase: !!str
  # An updated copy of the request body
  request:
    executable:
      ....
    resources:
      ....
    datetime:
      ....
\end{lstlisting}

\subsection{Extensions}
\label{sub-datamodel-extensions}

\begin{lstlisting}[]
....
....
\end{lstlisting}

\subsubsection{nvidia gpu}
\label{subsub-datamodel-nvidia-gpu}

\begin{lstlisting}[]
....
....
\end{lstlisting}

\subsubsection{Xilinx FPGA}
\label{subsub-datamodel-xilinx-fpga}

\begin{lstlisting}[]
type: "https://www.purl.org/ivoa.net/resource-types/xilinx-vu19p"
# The details, specific to a 'Xilinx FPGA'.
....
....
\end{lstlisting}


\pagebreak
\appendix
\section{Changes from Previous Versions}
\label{sect-changes}

No previous versions yet.
% these would be subsections "Changes from v. WD-..."
% Use itemize environments.

\pagebreak
\appendix
\section{Resource type URLs}
\label{sect-type-urls}

https://www.purl.org/ivoa.net/executable-types/example
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/executable-types/example-executable.md

https://www.purl.org/ivoa.net/executable-types/jupyter-notebook
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/executable-types/jupyter-notebook.md

https://www.purl.org/ivoa.net/executable-types/oci-container
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/executable-types/oci-container.md

https://www.purl.org/ivoa.net/resource-types/generic-compute
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/resource-types/generic-compute.md

https://www.purl.org/ivoa.net/resource-types/nvidia-ad104
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/resource-types/nvidia-ad104.md

https://www.purl.org/ivoa.net/resource-types/xilinx-vu19p
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/resource-types/xilinx-vu19p.md

https://www.purl.org/ivoa.net/storage-types/ephemeral-storage
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/storage-types/ephemeral-storage.md

https://www.purl.org/ivoa.net/storage-types/amazon-s3
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/storage-types/amazon-s3.md

https://www.purl.org/ivoa.net/storage-types/persistent-storage
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/storage-types/persistent-storage.md

https://www.purl.org/ivoa.net/storage-types/rucio-storage
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/storage-types/rucio-storage.md

https://www.purl.org/ivoa.net/storage-types/vospace-storage
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/storage-types/vospace-storage.md

https://www.purl.org/ivoa.net/authentication-types/basic-auth
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/authentication-types/basic-auth.md

https://www.purl.org/ivoa.net/authentication-types/oidc-auth
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/authentication-types/oidc-auth.md

https://www.purl.org/ivoa.net/authentication-types/rfc7519-token
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/authentication-types/rfc7519-token.md

https://www.purl.org/ivoa.net/webservice-types/value-update
https://github.com/ivoa-std/ExecutionBroker/blob/main/types/webservice-types/value-update.md

YAML specification
https://yaml.org/spec/

YAML timestamp
https://yaml.org/type/timestamp.html

% NOTE: IVOA recommendations must be cited from docrepo rather than ivoabib
% (REC entries there are for legacy documents only)
\bibliography{ivoatex/ivoabib,ivoatex/docrepo}


\end{document}