Merge branch 'master' into feature/SK-812

.github/workflows/code-checks.yaml

@@ -26,6 +26,8 @@ jobs:
           --exclude-dir='flower-client'
           --exclude='tests.py'
           --exclude='controller_cmd.py'
+          --exclude='combiner_cmd.py'
+          --exclude='run_cmd.py'
           --exclude='README.rst'
           '^[ \t]+(import|from) ' -I . 
 

LICENSE

@@ -1,3 +1,5 @@
+Copyright 2021 Scaleout Systems AB.  All rights reserved.
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.rst
+include fedn/common/settings-controller.yaml.template

config/settings-client.yaml.local.template

@@ -0,0 +1,3 @@
+network_id: fedn-network
+discover_host: localhost
+discover_port: 8092

config/settings-combiner.yaml.local.template

@@ -0,0 +1,31 @@
+network_id: fedn-network
+
+name: combiner
+host: localhost
+address: localhost
+port: 12080
+max_clients: 30
+
+cert_path: tmp/server.crt
+key_path: tmp/server.key
+
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: localhost
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: localhost
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False 
+
+

config/settings-combiner.yaml.template

@@ -7,4 +7,23 @@ host: combiner
 port: 12080
 max_clients: 30
 
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: mongo
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: minio
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False
+
 

config/settings-controller.yaml.local.template

@@ -0,0 +1,24 @@
+network_id: fedn-network
+controller:
+  host: localhost
+  port: 8092
+  debug: True
+
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: localhost
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: localhost
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False 

docker-compose.yaml

@@ -87,6 +87,8 @@ services:
     environment:
       - PYTHONUNBUFFERED=0
       - GET_HOSTS_FROM=dns
+      - STATESTORE_CONFIG=/app/config/settings-combiner.yaml
+      - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml
     build:
       context: .
       args:

docs/_static/css/elements.css

@@ -19,7 +19,6 @@ article ul {
 
 .rst-content .section ul li,
 .rst-content .toctree-wrapper ul li,
-.rst-content section ul li,
 .wy-plain-list-disc li,
 article ul li {
     list-style: none;

docs/conf.py

@@ -8,11 +8,10 @@
 
 # Project info
 project = "FEDn"
-copyright = "2021, Scaleout Systems AB"
 author = "Scaleout Systems AB"
 
 # The full version, including alpha/beta/rc tags
-release = "0.12.1"
+release = "0.14.0"
 
 # Add any Sphinx extension module names here, as strings
 extensions = [
@@ -98,8 +97,7 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, "fedn", "FEDn Documentation", author, "fedn",
-     "One line description of project.", "Miscellaneous"),
+    (master_doc, "fedn", "FEDn Documentation", author, "fedn", "One line description of project.", "Miscellaneous"),
 ]
 
 # Bibliographic Dublin Core info.

docs/projects.rst

@@ -385,6 +385,15 @@ Then, standing inside the 'client folder', you can test *train* and *validate* b
     python train.py ../seed.npz ../model_update.npz --data_path data/clients/1/mnist.pt
     python validate.py ../model_update.npz ../validation.json --data_path data/clients/1/mnist.pt
 
+You can also test *train* and *validate* entrypoint using CLI command:
+
+.. note:: Before running the fedn run train or fedn run validate commands, make sure to download the training and test data. The downloads are usually handled by the "fedn run startup" command in the examples provided by FEDn.
+
+.. code-block:: bash
+
+    fedn run train --path client --input <path to input model parameters> --output <path to write the updated model parameters>
+    fedn run validate --path client --input <path to input model parameters> --output <path to write the output JSON containing validation metrics>
+
 Packaging for training on FEDn
 ===============================
 

docs/quickstart.rst

@@ -61,13 +61,13 @@ code used by the client to execute local training and local validation. The seed
 For a detailed explaination of the compute package and seed model, see this guide: :ref:`projects-label`
 
 To work through this quick start you need a local copy of the ``mnist-pytorch`` example project contained in the main FEDn Git repository. 
-The following command clones the entire repository but you will only use the example:
+Clone the repository using the following command, if you didn't already do it in the previous step:
 
 .. code-block:: bash
 
    git clone https://github.com/scaleoutsystems/fedn.git
 
-Locate into the ``fedn/examples/mnist-pytorch`` folder. The compute package is located in the folder ``client``.
+Navigate to the ``fedn/examples/mnist-pytorch`` folder. The compute package is located in the folder ``client``.
 
 Create a compute package: 
 
@@ -94,31 +94,39 @@ Next will now upload these files to your Studio project.
 
 3. Initialize the server-side
 ------------------------------
-The next step is to initialize the server side with the client code and the initial global model. 
-In the Studio UI, navigate to the project you created in step one and click on the "Sessions" tab. Click on the "New Session" button. Under the "Compute package" tab, select a name and upload the generated package file. Under the "Seed model" tab, upload the generated seed file:
+The next step is to initialize the server side with the client code and the initial global model. In the Studio UI,
+
+#. Navigate to the project you created in step 1 and click on the "Sessions" tab. 
+#. Click on the "New Session" button. 
+#. Under the "Compute package" tab, select a name and upload the generated package file. 
+#. Under the "Seed model" tab, upload the generated seed file:
 
 .. image:: img/upload_package.png
 
+Continue to step 4 before starting the session. The uploaded package and seed files are saved.
+
 4. Start clients
 -----------------
 
 Now we are ready to start FEDn clients on your local machine. There are two steps involved: 
 
-1. Register a new client in your Studio project, issuing an access token. 
-2. Start up a client process on your local host (using the token to connect securely)  
+#. Register a new client in your Studio project, issuing an access token. 
+#. Start up a client process on your local host (using the token to connect securely)  
 
 **Generate an access token for the client (in Studio)**
 
 Each local client needs an access token in order to connect securely to the FEDn server. These tokens are issued from your Studio Project. 
-Go to the Clients' tab and click 'Connect client'. Download a client configuration file and save it to the root of the ``examples/mnist-pytorch folder``. 
-Rename the file to 'client.yaml'. 
+
+#. Go to the 'Clients' tab and click 'Connect client'. 
+#. Download a client configuration file and save it to the root of the ``examples/mnist-pytorch`` folder. 
+#. Rename the file to 'client.yaml'. 
 
 **Start the client (on your local machine)** 
 
 The default training and test data for this particular example (mnist-pytorch) is for convenience downloaded and split automatically by the client when it starts up. 
 The number of splits and which split to use by a client can be controlled via the environment variables ``FEDN_NUM_DATA_SPLITS`` and ``FEDN_DATA_PATH``.
 
-Start a client (using a 10-split and the first partition) by running the following commands:
+Start a client (using a 10-split and the 1st partition) by running the following commands:
 
 .. tabs::
 
@@ -139,20 +147,25 @@ Start a client (using a 10-split and the first partition) by running the followi
          fedn run client -in client.yaml --secure=True --force-ssl
 
 Repeat these two steps (generate an access token and start a local client) for the number of clients you want to use.
-A normal laptop should be able to handle several clients for this example. Remember to use different partitions for each client. 
+A normal laptop should be able to handle several clients for this example. Remember to use different partitions for each client, by changing the number in the ``FEDN_DATA_PATH`` variable. 
 
 5. Train the global model 
 -----------------------------
 
 With clients connected, we are now ready to train the global model. This can be done using either the Studio dashboard or the Python API. In FEDn, training is organised 
 in Sessions. One training session consists of a configurable number of training rounds (local model updates and aggregation).  
 
-In Studio click on the "Sessions" link, then the "New session" button in the upper right corner. Click the "Start session" tab and enter your desirable settings (the default settings are good for this example) and hit the "Start run" button.
+In Studio, 
+
+#. Click on the "Sessions" link, then the "New session" button in the upper right corner. 
+#. Click the "Start session" tab and enter your desirable settings (the default settings are good for this example).
+#. Hit the "Start run" button.
+
 In the terminal where your are running your client you should now see some activity. When a round is completed, you can see the results on the "Models" page.
 
 **Watch real-time updates of training progress**
 
-Once a training session is started, you can monitor the progress by clicking the drop-down button for the active Sessions and the clicking on the "View session" button. The session page will show 
+Once a training session is started, you can monitor the progress by clicking the drop-down button for the active Sessions and then clicking on the "View session" button. The session page will show 
 metrics related to the training progress (accuracy, loss etc), as well as performance data such as total round times and individual client training times. A list of models in the session is updated as soon as new models are generated. 
 To get more information about a particular model, navigate to the model page by clicking the model name. From the model page you can download the model weights and get validation metrics.
 
@@ -173,19 +186,19 @@ be useful as you progress in your federated learning journey.
 With you first FEDn federated project set up, we suggest that you take a closer look at how a FEDn project is structured
 to learn how to develop your own FEDn projects:
 
-- :ref:`projects-label`
+:ref:`projects-label`
 
 In this tutorial we relied on the UI for running training sessions and retrieving models and results. 
 The Python APIClient provides a flexible alternative, with additional functionality exposed, 
 including the use of different aggregators. Learn how to use the APIClient here: 
 
-- :ref:`apiclient-label`
+:ref:`apiclient-label`
 
 Study the architecture overview to learn more about how FEDn is designed and works under the hood: 
 
-- :ref:`architecture-label`
+:ref:`architecture-label`
 
 For developers looking to customize FEDn and develop own aggregators, check out the local development guide
 to learn how to set up an all-in-one development environment using Docker and docker-compose:
 
-- :ref:`developer-label`
+:ref:`developer-label`

examples/mnist-pytorch/client/python_env.yaml

@@ -4,6 +4,6 @@ build_dependencies:
   - setuptools
   - wheel
 dependencies:
-  - torch==2.3.1
-  - torchvision==0.18.1
+  - torch
+  - torchvision
   - fedn

fedn/cli/client_cmd.py

@@ -6,8 +6,8 @@
 from fedn.common.exceptions import InvalidClientConfig
 from fedn.network.clients.client import Client
 
-from .main import main
-from .shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response
+from fedn.cli.main import main
+from fedn.cli.shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response
 
 
 def validate_client_config(config):

fedn/cli/combiner_cmd.py

@@ -3,17 +3,14 @@
 import click
 import requests
 
-from fedn.network.combiner.combiner import Combiner
-
 from .main import main
 from .shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response
 
 
 @main.group("combiner")
 @click.pass_context
 def combiner_cmd(ctx):
-    """:param ctx:
-    """
+    """:param ctx:"""
     pass
 
 
@@ -60,6 +57,8 @@ def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, se
         click.echo(f"\nCombiner configuration loaded from file: {init}")
         click.echo("Values set in file override defaults and command line arguments...\n")
 
+    from fedn.network.combiner.combiner import Combiner
+
     combiner = Combiner(config)
     combiner.run()
 

fedn/cli/controller_cmd.py

@@ -1,6 +1,6 @@
 import click
 
-from .main import main
+from fedn.cli.main import main
 
 
 @main.group("controller")