6.2.configuration to deploy mlflow

Makefile

@@ -19,6 +19,8 @@ VALIDATION_SERVICE_NAME=punch-validation.service
 VALIDATION_TIMER_NAME=punch-validation.timer
 VALIDATION_SERVICE_SCRIPT=${PUNCHBOX_SCRIPT_DIR}/${VALIDATION_SERVICE_NAME}
 VALIDATION_TIMER_SCRIPT=${PUNCHBOX_SCRIPT_DIR}/${VALIDATION_TIMER_NAME}
+MLFLOW_SERVER=$(shell jq -r ".mlflow.servers | to_entries[0].key" punch/build/pp-conf/punchplatform-deployment.settings)
+
 
 VENV_MARKERFILE=${DIR}/.venv/.installed
 DEPENDENCIES_INSTALLED_MARKERFILE=${DIR}/vagrant/.dependencies_installed
@@ -239,6 +241,7 @@ local-integration-vagrant:  ## Use this rule instead of deploy-config if you are
 	@. ${ACTIVATE_SH} && punchplatform-deployer.sh -cp -u vagrant
 	@$(call green, "Check if vagrant boxes are up", "")
 	@cd ${DIR}/vagrant && ${VAGRANT} up
+	@cd ${DIR}/vagrant &&  ${VAGRANT} ssh ${MLFLOW_SERVER} -c "/data/mlflow_provisionner_src/provision-mlflow.sh; exit"
 	@$(call green, "Executing on server1", "/home/vagrant/pp-conf/check_platform.sh")
 	@cd ${DIR}/vagrant && ${VAGRANT} ssh server1 -c "/home/vagrant/pp-conf/check_platform.sh; exit"
 

bin/punchbox.py

@@ -38,7 +38,7 @@
 vagrantfile_target = vagrant_dir + '/Vagrantfile'
 generated_model = build_dir + '/model.json'
 
-cots = ["punch", "minio", "zookeeper", "spark", "elastic", "opendistro_security", "operator", "binaries",
+cots = ["punch", "minio", "mlflow", "zookeeper", "spark", "elastic", "opendistro_security", "operator", "binaries",
         "analytics-deployment",
         "analytics-client", "shiva", "gateway", "storm", "kafka", "logstash", "metricbeat", "filebeat", "packetbeat",
         "auditbeat"]
@@ -95,7 +95,7 @@ def create_vagrantfile(platform_config, vagrant_os: str=None):
   file_loader = jinja2.FileSystemLoader(vagrant_dir)
   env = jinja2.Environment(loader=file_loader)
   vagrantfile_template = env.get_template(vagrant_template_file)
-  vagrantfile_render = vagrantfile_template.render(targets=platform_config["targets"], os=vagrant_os)
+  vagrantfile_render = vagrantfile_template.render(targets=platform_config["targets"], os=vagrant_os, punch=platform_config["punch"])
   vagrantfile = open(vagrantfile_target, "w+")
   vagrantfile.write(vagrantfile_render)
   vagrantfile.close()

configurations/README.md

@@ -66,9 +66,15 @@ This file is composed as follow :
       - `operator`: 
           - `servers`: list of operator hosts
           - `username`: operator username
+      - `mlflow`: 
+	  - `servers`: list of mlflow servers
+	     - `servers_name` : name of the current server
+	       - `bind_adress` : Binding adress 
+	       - `logs_path `: Path where to store mlflow logs (must be reachable by server side)
+	       - `artifacts_path` : Path where to store mlflow artifacts (must be reachable by server and client side)
 
 
 
 **Note** : All parameters under `targets` key are mandatory. For those under `punch`, they are optional
 
-**Note** : do **never** add or change things in the platform_template or vagrant without a first review with the core punch team leaders
+**Note** : do **never** add or change things in the platform_template or vagrant without a first review with the core punch team leaders

configurations/complete_punch_16G.json

@@ -36,6 +36,22 @@
         "server1"
       ]
     },
+    "mlflow": {
+        "servers": {
+		"server1" : {
+			"bind_address" : "0.0.0.0",
+			"port" : "5000",
+			"logs_path" : "/tmp/mlflow-logs",
+			"artifacts_path" : "S3://mlflow-artifacts"
+		}
+        },
+	"provisioning_src_path": "/data/mlflow_provisionner_src/"
+    },
+    "minio" : {
+	"servers" : [
+		"server2"
+	]
+    },
     "zookeeper": {
       "servers": [
         "server3"
@@ -95,4 +111,4 @@
       "username": "vagrant"
     }
   }
-}
+}

configurations/complete_punch_32G.json

@@ -46,6 +46,22 @@
         "server1"
       ]
     },
+    "mlflow": {
+        "servers": {
+		"server1" : {
+			"bind_address" : "0.0.0.0",
+			"port" : "5000",
+			"logs_path" : "/tmp/mlflow-logs",
+			"artifacts_path" : "S3://mlflow-artifacts"
+		}
+        },
+	"provisioning_src_path": "/data/mlflow_provisionner_src/"
+    },
+    "minio" : {
+	"servers" : [
+		"server2"
+	]
+    },
     "zookeeper": {
       "servers": [
         "server2"

punch/deployment_template/punchplatform-deployment.settings.j2

@@ -350,6 +350,25 @@
     },
    {% endif %}
 
+   {% if mlflow is defined %}
+    "mlflow" : {
+     "version" : "{{ version.mlflow }}",
+     "servers" : {
+    {% for server_id, server_section in mlflow.servers.items() %}
+      "{{ server_id }}" : {
+        "bind_address" : "{{ server_section.bind_address }}",
+        "port" : "{{ server_section.port }}",
+        "logs_path" : "{{ server_section.logs_path }}",
+        "artifacts_path" : "{{ server_section.artifacts_path }}"
+      }
+      {% if not loop.last %}
+      ,
+      {% endif %}
+    {% endfor %}
+    }
+    },
+   {% endif %}
+
    {% if clickhouse is defined %}
   "clickhouse": {
     "clickhouse_version": "20.4.6.53",

vagrant/Vagrantfile.j2

@@ -86,6 +86,19 @@ Vagrant.configure("2") do |config|
       server.vm.provision "shell", inline: "echo '172.28.128.2{{ loop.index }} {{ server }}' >> /etc/hosts"
       {%- endfor %}
 
+      {% if punch.mlflow is defined %}
+            {%- for mlflowServerName, mlflowServerAttributes in punch.mlflow.servers.items() %}
+		{%- if mlflowServerName == server %}
+      server.vm.provision "ansible" do |ansible| 
+	ansible.become = true
+        ansible.playbook = "ansible-resources/vagrant-playbook.yml"
+	ansible.extra_vars= { "punchbox_daemon_user": "vagrant", "punch": {{punch | tojson}} }
+      end
+		{%- endif %}
+            {%- endfor %}
+      {%- endif %}
+
+
       server.vm.provider :virtualbox do |v|
         v.gui = false
         v.memory = {{ targets.info[server].memory }}