MLFlow migration

wmgeolab · Dec 17, 2024 · 8eeefac · 8eeefac
1 parent 514211f
commit 8eeefac
Show file tree

Hide file tree

Showing 36 changed files with 336 additions and 229 deletions.
diff --git a/geoBoundaryBuilder/baseline_ex.py b/geoBoundaryBuilder/baseline_ex.py
diff --git a/geoBoundaryBuilder/flows/A_pull_from_github/git_sync.py b/geoBoundaryBuilder/flows/A_pull_from_github/git_sync.py
@@ -0,0 +1,67 @@
+import mlflow
+import subprocess
+import os
+import sys
+
+# MLflow Tracking Configuration
+MLFLOW_TRACKING_URI = "http://mlflow-server-service.geoboundaries.svc.cluster.local:5000"
+mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)
+
+# Set Experiment Name
+EXPERIMENT_NAME = "Pull from Github"
+mlflow.set_experiment(EXPERIMENT_NAME)
+
+# Constants
+SSH_DIR = "/sciclone/geograd/geoBoundaries/.ssh"
+GIT_REPO_DIR = "/sciclone/geograd/geoBoundaries/database/geoBoundaries"
+KNOWN_HOSTS = os.path.join(SSH_DIR, "known_hosts")
+SSH_KEY = os.path.join(SSH_DIR, "id_ed25519")
+HOME_DIR = "/sciclone/geograd/geoBoundaries"
+
+def git_pull():
+    """Executes git pull and logs results to MLflow."""
+    with mlflow.start_run():
+        mlflow.log_param("operation", "git pull")
+        try:
+            # Set up SSH directory
+            os.makedirs(SSH_DIR, exist_ok=True)
+            os.chmod(SSH_DIR, 0o700)
+            subprocess.run(["ssh-keyscan", "-H", "github.com"], stdout=open(KNOWN_HOSTS, "a"), check=True)
+            os.chmod(KNOWN_HOSTS, 0o600)
+
+            # Inline git config and git pull
+            print("Starting git pull...")
+            git_command = (
+                f"git -C {GIT_REPO_DIR} config --global --add safe.directory {GIT_REPO_DIR} && "
+                f"git -C {GIT_REPO_DIR} pull"
+            )
+            result = subprocess.run(
+                git_command,
+                env={
+                    "HOME": HOME_DIR,
+                    "GIT_SSH_COMMAND": f"ssh -i {SSH_KEY} -o UserKnownHostsFile={KNOWN_HOSTS}"
+                },
+                shell=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+
+            if result.returncode == 0:
+                print("Git pull succeeded.")
+                mlflow.log_metric("status", 1)
+                mlflow.log_param("git_stdout", result.stdout)
+            else:
+                print("Git pull failed.", file=sys.stderr)
+                mlflow.log_metric("status", 0)
+                mlflow.log_param("git_stderr", result.stderr)
+                mlflow.log_param("git_stdout", result.stdout)
+                sys.exit(1)
+        except Exception as e:
+            print(f"Error occurred: {e}", file=sys.stderr)
+            mlflow.log_metric("status", 0)
+            mlflow.log_param("error", str(e))
+            sys.exit(1)
+
+if __name__ == "__main__":
+    git_pull()
diff --git a/geoBoundaryBuilder/flows/A_pull_from_github/mlruns/0/meta.yaml b/geoBoundaryBuilder/flows/A_pull_from_github/mlruns/0/meta.yaml
@@ -0,0 +1,6 @@
+artifact_location: file:///sciclone/geograd/geoBoundaries/geoBoundaryBot/geoBoundaryBuilder/flows/pull_from_github/mlruns/0
+creation_time: 1734458796287
+experiment_id: '0'
+last_update_time: 1734458796287
+lifecycle_stage: active
+name: Default
diff --git a/...s/A_pull_from_github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/meta.yaml b/...s/A_pull_from_github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/meta.yaml
@@ -0,0 +1,15 @@
+artifact_uri: file:///sciclone/geograd/geoBoundaries/geoBoundaryBot/geoBoundaryBuilder/flows/pull_from_github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/artifacts
+end_time: null
+entry_point_name: ''
+experiment_id: '119041156989095158'
+lifecycle_stage: active
+run_id: 412a017eb320458f98905c5f6dc7c41c
+run_name: enthused-snail-24
+run_uuid: 412a017eb320458f98905c5f6dc7c41c
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1734459181050
+status: 1
+tags: []
+user_id: root
diff --git a/..._github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.gitRepoURL b/..._github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.gitRepoURL
@@ -0,0 +1 @@
+http://github.com/wmgeolab/geoBoundaryBot
diff --git a/...mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.project.entryPoint b/...mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.project.entryPoint
@@ -0,0 +1 @@
+main
diff --git a/...github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.project.env b/...github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.project.env
@@ -0,0 +1 @@
+virtualenv
diff --git a/...rom_github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.runName b/...rom_github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.runName
@@ -0,0 +1 @@
+enthused-snail-24
diff --git a/...mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.source.git.repoURL b/...mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.source.git.repoURL
@@ -0,0 +1 @@
+http://github.com/wmgeolab/geoBoundaryBot
diff --git a/...github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.source.name b/...github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.source.name
@@ -0,0 +1 @@
+file:///sciclone/geograd/geoBoundaries/geoBoundaryBot#geoBoundaryBuilder/flows/pull_from_github
diff --git a/...github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.source.type b/...github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.source.type
@@ -0,0 +1 @@
+PROJECT
diff --git a/...l_from_github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.user b/...l_from_github/mlruns/119041156989095158/412a017eb320458f98905c5f6dc7c41c/tags/mlflow.user
@@ -0,0 +1 @@
+root
diff --git a/...s/A_pull_from_github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/meta.yaml b/...s/A_pull_from_github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/meta.yaml
@@ -0,0 +1,15 @@
+artifact_uri: file:///sciclone/geograd/geoBoundaries/geoBoundaryBot/geoBoundaryBuilder/flows/pull_from_github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/artifacts
+end_time: null
+entry_point_name: ''
+experiment_id: '119041156989095158'
+lifecycle_stage: active
+run_id: 9c573668c4804940bf7e9e93d8f0fbe4
+run_name: painted-moth-181
+run_uuid: 9c573668c4804940bf7e9e93d8f0fbe4
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1734459110670
+status: 1
+tags: []
+user_id: root
diff --git a/..._github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.gitRepoURL b/..._github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.gitRepoURL
@@ -0,0 +1 @@
+http://github.com/wmgeolab/geoBoundaryBot
diff --git a/...mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.project.entryPoint b/...mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.project.entryPoint
@@ -0,0 +1 @@
+main
diff --git a/...github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.project.env b/...github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.project.env
@@ -0,0 +1 @@
+virtualenv
diff --git a/...rom_github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.runName b/...rom_github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.runName
@@ -0,0 +1 @@
+painted-moth-181
diff --git a/...mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.source.git.repoURL b/...mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.source.git.repoURL
@@ -0,0 +1 @@
+http://github.com/wmgeolab/geoBoundaryBot
diff --git a/...github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.source.name b/...github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.source.name
@@ -0,0 +1 @@
+file:///sciclone/geograd/geoBoundaries/geoBoundaryBot#geoBoundaryBuilder/flows/pull_from_github
diff --git a/...github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.source.type b/...github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.source.type
@@ -0,0 +1 @@
+PROJECT
diff --git a/...l_from_github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.user b/...l_from_github/mlruns/119041156989095158/9c573668c4804940bf7e9e93d8f0fbe4/tags/mlflow.user
@@ -0,0 +1 @@
+root
diff --git a/...s/A_pull_from_github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/meta.yaml b/...s/A_pull_from_github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/meta.yaml
@@ -0,0 +1,15 @@
+artifact_uri: file:///sciclone/geograd/geoBoundaries/geoBoundaryBot/geoBoundaryBuilder/flows/pull_from_github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/artifacts
+end_time: null
+entry_point_name: ''
+experiment_id: '119041156989095158'
+lifecycle_stage: active
+run_id: af6bd54fc28f4cb3b016c215e630d07c
+run_name: bald-cub-840
+run_uuid: af6bd54fc28f4cb3b016c215e630d07c
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1734458887978
+status: 1
+tags: []
+user_id: root
diff --git a/..._github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.gitRepoURL b/..._github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.gitRepoURL
@@ -0,0 +1 @@
+http://github.com/wmgeolab/geoBoundaryBot
diff --git a/...mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.project.entryPoint b/...mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.project.entryPoint
@@ -0,0 +1 @@
+main
diff --git a/...github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.project.env b/...github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.project.env
@@ -0,0 +1 @@
+virtualenv
diff --git a/...rom_github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.runName b/...rom_github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.runName
@@ -0,0 +1 @@
+bald-cub-840
diff --git a/...mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.source.git.repoURL b/...mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.source.git.repoURL
@@ -0,0 +1 @@
+http://github.com/wmgeolab/geoBoundaryBot
diff --git a/...github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.source.name b/...github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.source.name
@@ -0,0 +1 @@
+file:///sciclone/geograd/geoBoundaries/geoBoundaryBot#geoBoundaryBuilder/flows/pull_from_github
diff --git a/...github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.source.type b/...github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.source.type
@@ -0,0 +1 @@
+PROJECT
diff --git a/...l_from_github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.user b/...l_from_github/mlruns/119041156989095158/af6bd54fc28f4cb3b016c215e630d07c/tags/mlflow.user
@@ -0,0 +1 @@
+root
diff --git a/geoBoundaryBuilder/flows/A_pull_from_github/mlruns/119041156989095158/meta.yaml b/geoBoundaryBuilder/flows/A_pull_from_github/mlruns/119041156989095158/meta.yaml
@@ -0,0 +1,6 @@
+artifact_location: file:///sciclone/geograd/geoBoundaries/geoBoundaryBot/geoBoundaryBuilder/flows/pull_from_github/mlruns/119041156989095158
+creation_time: 1734458796385
+experiment_id: '119041156989095158'
+last_update_time: 1734458796385
+lifecycle_stage: active
+name: Pull from Github
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		file:///sciclone/geograd/geoBoundaries/geoBoundaryBot#geoBoundaryBuilder/flows/pull_from_github