Add prometheus metrics support via texfile collector

Cargo.toml

@@ -21,3 +21,4 @@ serde_derive = "1.0"
 serde_json = "1.0"
 serde_yaml = "0.7.0"
 fs2 = "0.4.2"
+prometheus = "0.2.8"

src/lib.rs

@@ -7,6 +7,7 @@
 use std::process::{Command, Stdio, exit};
 use std::fs;
 use std::path::Path;
+use std::fs::File;
 
 // File locking
 extern crate fs2;
@@ -15,7 +16,7 @@ use fs2::FileExt;
 // Used for error and debug logging
 #[macro_use]
 extern crate log;
-use log::LogLevel::Info;
+use log::LogLevel::{Debug, Info};
 
 // Used to create sane local directory names
 extern crate slug;
@@ -36,7 +37,12 @@ use std::sync::mpsc::channel;
 
 // Time handling
 extern crate chrono;
-use chrono::Local;
+use chrono::{Local, Utc};
+
+// Monitoring
+#[macro_use]
+extern crate prometheus;
+use prometheus::{TextEncoder, Encoder};
 
 use provider::{Mirror, Provider};
 
@@ -58,9 +64,8 @@ pub fn mirror_repo(
     // Group common setting for al git commands in this closure
     let git_base_cmd = || {
         let mut git = Command::new("git");
-        if !log_enabled!(Info) {
+        if !log_enabled!(Debug) {
             git.stdout(Stdio::null());
-            git.stderr(Stdio::null());
         }
         debug!("Level {:?}", log_enabled!(Info));
         git.env("GIT_TERMINAL_PROMPT", "0");
@@ -191,20 +196,44 @@ fn run_sync_task(v: Vec<Mirror>, worker_count: usize, mirror_dir: &str, dry_run:
     let pool = ThreadPool::new(worker_count);
     let mut n = 0;
 
+    let proj_fail = register_counter!("git_mirror_fail", "Failed projects").unwrap();
+    let proj_ok = register_counter!("git_mirror_ok", "OK projects").unwrap();
+    let proj_start = register_gauge_vec!(
+        "git_mirror_project_start",
+        "Start of project mirror as unix timestamp",
+        &["origin", "destination"]
+    ).unwrap();
+    let proj_end = register_gauge_vec!(
+        "git_mirror_project_end",
+        "End of projeect mirror as unix timestamp",
+        &["origin", "destination"]
+    ).unwrap();
+
     let (tx, rx) = channel();
     for x in v {
         let tx = tx.clone();
         let mirror_dir = mirror_dir.to_owned().clone();
+        let proj_fail = proj_fail.clone();
+        let proj_ok = proj_ok.clone();
+        let proj_start = proj_start.clone();
+        let proj_end = proj_end.clone();
         pool.execute(move || {
             println!(
                 "START [{}]: {} -> {}",
                 Local::now(),
                 x.origin,
                 x.destination
             );
+            proj_start
+                .with_label_values(&[&x.origin, &x.destination])
+                .set(Utc::now().timestamp() as f64);
             let c = match mirror_repo(mirror_dir, &x.origin, &x.destination, dry_run) {
                 Ok(c) => {
                     println!("OK [{}]: {} -> {}", Local::now(), x.origin, x.destination);
+                    proj_end
+                        .with_label_values(&[&x.origin, &x.destination])
+                        .set(Utc::now().timestamp() as f64);
+                    proj_ok.inc();
                     c
                 }
                 Err(e) => {
@@ -215,6 +244,10 @@ fn run_sync_task(v: Vec<Mirror>, worker_count: usize, mirror_dir: &str, dry_run:
                         x.destination,
                         e
                     );
+                    proj_end
+                        .with_label_values(&[&x.origin, &x.destination])
+                        .set(Utc::now().timestamp() as f64);
+                    proj_fail.inc();
                     error!(
                         "Unable to sync repo {} -> {} ({})",
                         x.origin,
@@ -239,7 +272,21 @@ fn run_sync_task(v: Vec<Mirror>, worker_count: usize, mirror_dir: &str, dry_run:
 }
 
 
-pub fn do_mirror(provider: &Provider, worker_count: usize, mirror_dir: &str, dry_run: bool) {
+pub fn do_mirror(
+    provider: &Provider,
+    worker_count: usize,
+    mirror_dir: &str,
+    dry_run: bool,
+    metrics_file: Option<String>,
+) {
+    let start_time = register_gauge!(
+        "git_mirror_start_time",
+        "Start time of the sync as unix timestamp"
+    ).unwrap();
+    let end_time = register_gauge!(
+        "git_mirror_end_time",
+        "End time of the sync as unix timestamp"
+    ).unwrap();
 
     // Make sure the mirror directory exists
     trace!("Create mirror directory at {:?}", mirror_dir);
@@ -272,9 +319,24 @@ pub fn do_mirror(provider: &Provider, worker_count: usize, mirror_dir: &str, dry
         exit(1);
     });
 
+    start_time.set(Utc::now().timestamp() as f64);
+
     run_sync_task(v, worker_count, mirror_dir, dry_run);
 
+    end_time.set(Utc::now().timestamp() as f64);
+
+    match metrics_file {
+        Some(f) => write_metrics(&f),
+        None => trace!("Skipping merics file creation"),
+
+    };
 }
 
+fn write_metrics(f: &str) {
+    let mut file = File::create(f).unwrap();
+    let encoder = TextEncoder::new();
+    let metric_familys = prometheus::gather();
+    encoder.encode(&metric_familys, &mut file).unwrap();
+}
 
 pub mod provider;

src/main.rs

@@ -83,6 +83,15 @@ fn main() {
                 .possible_values(&Providers::variants())
                 .default_value("GitLab"),
         )
+        .arg(
+            Arg::with_name("metrics-file")
+                .long("metrics-file")
+                .help(
+                    "Location where to store metrics for consumption by \
+                       Prometheus nodeexporter's text file colloctor.",
+                )
+                .takes_value(true),
+        )
         .after_help(
             "ENVIRONMENT:\n    GITLAB_PRIVATE_TOKEN    \
                      Private token or Personal access token to access the GitLab API",
@@ -111,6 +120,8 @@ fn main() {
     debug!("Dry run: {}", dry_run);
     let worker_count = value_t_or_exit!(m.value_of("worker-count"), usize);
     debug!("Worker count: {}", worker_count);
+    let metrics_file = value_t!(m.value_of("metrics-file"), String).ok();
+    debug!("Metrics file: {:?}", metrics_file);
 
     let provider = value_t_or_exit!(m.value_of("provider"), Providers);
 
@@ -122,7 +133,7 @@ fn main() {
                 use_http: use_http,
                 private_token: gitlab_private_token,
             };
-            do_mirror(&p, worker_count, &mirror_dir, dry_run);
+            do_mirror(&p, worker_count, &mirror_dir, dry_run, metrics_file);
         }
         Providers::GitHub => {
             let p = GitHub {
@@ -132,7 +143,7 @@ fn main() {
                 private_token: gitlab_private_token,
                 useragent: format!("{}/{}", crate_name!(), crate_version!()),
             };
-            do_mirror(&p, worker_count, &mirror_dir, dry_run);
+            do_mirror(&p, worker_count, &mirror_dir, dry_run, metrics_file);
         }
     };
 }