initial commit

.ci/tests/examples/configure.sh

@@ -13,8 +13,14 @@ pushd "examples/$example"
 bin/init_venv.sh
 
 >&2 echo "Download and prepare data"
-bin/get_data
-bin/split_data
+# Check if $example is "multiple_models" 
+if [ "$example" == "multiple_models" ]; then
+    bin/get_data.sh
+    bin/split_data.sh
+else
+    bin/get_data
+    bin/split_data
+fi
 
 >&2 echo "Build compute package and seed"
 bin/build.sh

examples/mnist-keras/client/fedn.yaml

@@ -1,7 +1,9 @@
 entry_points:
   train:
-    command: /venv/bin/python entrypoint train $ENTRYPOINT_OPTS
+    default: /venv/bin/python entrypoint train $ENTRYPOINT_OPTS
+    test: /venv/bin/python entrypoint train $ENTRYPOINT_OPTS
   validate:
-    command: /venv/bin/python entrypoint validate $ENTRYPOINT_OPTS
+    default: /venv/bin/python entrypoint validate $ENTRYPOINT_OPTS
+    test: /venv/bin/python entrypoint validate $ENTRYPOINT_OPTS
   infer:
-    command: /venv/bin/python entrypoint infer $ENTRYPOINT_OPTS
+    default: /venv/bin/python entrypoint infer $ENTRYPOINT_OPTS

examples/mnist-keras/requirements.txt

@@ -1,3 +1,3 @@
-tensorflow==2.9.3
+tensorflow-cpu==2.14.0
 fire==0.3.1
 docker==6.1.1

examples/multiple_models/.gitignore

@@ -0,0 +1,6 @@
+data
+*.npz
+*.tgz
+*.tar.gz
+.multiple_models
+client.yaml

examples/multiple_models/README.md

@@ -0,0 +1,87 @@
+# MNIST (TensorFlow/Keras version)
+This classic example of hand-written text recognition is well suited both as a lightweight test when developing on FEDn in pseudo-distributed mode. A normal high-end laptop or a workstation should be able to sustain a few clients. The example automates the partitioning of data and deployment of a variable number of clients on a single host. We here assume working experience with containers, Docker and docker-compose. 
+
+## Table of Contents
+- [MNIST Example (Keras version)](#mnist-example-keras-version)
+  - [Table of Contents](#table-of-contents)
+  - [Prerequisites](#prerequisites)
+  - [Running the example (pseudo-distributed)](#running-the-example-pseudo-distributed)
+  - [Clean up](#clean-up)
+
+## Prerequisites
+- [Python 3.8, 3.9 or 3.10](https://www.python.org/downloads)
+- [Docker](https://docs.docker.com/get-docker)
+- [Docker Compose](https://docs.docker.com/compose/install)
+
+## Running the example (pseudo-distributed)
+Clone FEDn and locate into this directory.
+```sh
+git clone https://github.com/scaleoutsystems/fedn.git
+cd fedn/examples/mnist-keras
+```
+
+### Preparing the environment, the local data, the compute package and seed model
+
+Start by initializing a virtual enviroment with all of the required dependencies.
+```sh
+bin/init_venv.sh
+```
+
+Then, to get the data you can run the following script.
+```sh
+bin/get_data
+```
+
+The next command splits the data in 2 parts for the clients.
+```sh
+bin/split_data
+```
+> **Note**: run with `--n_splits=N` to split in *N* parts.
+
+Create the compute package and a seed model that you will be asked to upload in the next step.
+```
+bin/build.sh
+```
+> The files location will be `package/package.tgz` and `seed.npz`.
+
+### Deploy FEDn 
+Now we are ready to deploy FEDn with `docker-compose`.
+```
+docker-compose -f ../../docker-compose.yaml up -d minio mongo mongo-express reducer combiner
+```
+
+### Initialize the federated model 
+Now navigate to http://localhost:8090 to see the reducer UI. You will be asked to upload the compute package and the seed model that you created in the previous step.
+
+### Attach clients 
+To attach clients to the network, use the docker-compose.override.yaml template to start 2 clients: 
+
+```
+docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml up client 
+```
+> **Note**: run with `--scale client=N` to start *N* clients.
+
+### Run federated training 
+Finally, you can start the experiment from the "control" tab of the UI. 
+
+## Clean up
+You can clean up by running `docker-compose down`.
+
+## Connecting to a distributed deployment
+To start and remotely connect a client with the required dependencies for this example, start by downloading the `client.yaml` file. You can either navigate the reducer UI or run the following command.
+
+```bash
+curl -k https://<reducer-fqdn>:<reducer-port>/config/download > client.yaml
+```
+> **Note** make sure to replace `<reducer-fqdn>` and `<reducer-port>` with appropriate values.
+
+Now you are ready to start the client via Docker by running the following command.
+
+```bash
+docker run -d \
+  -v $PWD/client.yaml:/app/client.yaml \
+  -v $PWD/data:/var/data \
+  -e ENTRYPOINT_OPTS=--data_path=/var/data/mnist.npz \
+  ghcr.io/scaleoutsystems/fedn/fedn:develop-mnist-keras run client -in client.yaml
+```
+> **Note** If reducer and combiner host names, as specfied in the configuration files, are not resolvable in the client host network you need to use the docker option `--add-hosts` to make them resolvable. Please refer to the Docker documentation for more detail.

examples/multiple_models/bin/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+# Init for each model
+#pushd ../mnist-keras
+#client/entrypoint init_seed
+#popd
+
+#pushd ../mnist-pytorch
+#client/entrypoint init_seed
+#popd
+
+# Make compute package
+cp -r ../mnist-keras/client client/keras-client
+cp -r ../mnist-pytorch/client client/pytorch-client
+
+tar -czvf package.tgz client
+
+# Clean up
+rm -rf client/keras-client
+rm -rf client/pytorch-client

examples/multiple_models/bin/get_data.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+# Get the data for the first model
+./.multiple_models/bin/python ../mnist-keras/bin/get_data
+
+# Get the data for the second model
+./.multiple_models/bin/python ../mnist-pytorch/bin/get_data

examples/multiple_models/bin/init_venv.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+# Init venv
+python3 -m venv .multiple_models
+
+# Pip deps
+.multiple_models/bin/pip install --upgrade pip
+.multiple_models/bin/pip install -e ../../fedn
+.multiple_models/bin/pip install -r requirements.txt

examples/multiple_models/bin/split_data.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+# Split data for the first model
+./.multiple_models/bin/python ../mnist-keras/bin/split_data
+
+# Split data for the second model
+./.multiple_models/bin/python ../mnist-pytorch/bin/split_data

examples/multiple_models/client/fedn.yaml

@@ -0,0 +1,10 @@
+entry_points:
+  train:
+    keras: /venv/bin/python keras-client/entrypoint train $ENTRYPOINT_OPTS
+    pytorch: /venv/bin/python pytorch-client/entrypoint train $ENTRYPOINT_OPTS
+  validate:
+    keras: /venv/bin/python keras-client/entrypoint validate $ENTRYPOINT_OPTS
+    pytorch: /venv/bin/python pytorch-client/entrypoint validate $ENTRYPOINT_OPTS
+  infer:
+    keras: /venv/bin/python keras-client/entrypoint infer $ENTRYPOINT_OPTS
+    pytorch: /venv/bin/python pytorch-client/entrypoint infer $ENTRYPOINT_OPTS

examples/multiple_models/docker-compose.override.yaml

@@ -0,0 +1,15 @@
+# Compose schema version
+version: '3.3'
+
+# Overriding requirements
+services:
+  client:
+    build:
+      args:
+        REQUIREMENTS: examples/multiple_models/requirements.txt
+    deploy:
+      replicas: 2
+    volumes:
+      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+      - ${HOST_REPO_DIR:-.}/examples/multiple_models/data:/var/data
+      - /var/run/docker.sock:/var/run/docker.sock

examples/multiple_models/requirements.txt

@@ -0,0 +1,5 @@
+tensorflow-cpu==2.13.1
+torch==1.13.1
+torchvision==0.14.1
+fire==0.3.1
+docker==6.1.1

fedn/fedn/common/net/grpc/fedn.proto

@@ -54,6 +54,7 @@ message ModelUpdateRequest {
   string correlation_id = 5;
   string timestamp = 6;
   string meta = 7;
+  string model_tag = 8;
 }
 
 message ModelUpdate {
@@ -75,6 +76,7 @@ message ModelValidationRequest {
   string timestamp = 6; 
   string meta = 7;
   bool is_inference = 8;
+  string model_tag = 9;
 }
 
 message ModelValidation {