Fix for gaps in prometheus metrics.

README.md

@@ -44,7 +44,7 @@ The program can be used as follows:
 cronmanager -c command -n jobname [ -t time in seconds ] [ -l log file ]
 ```
 
-The `command`is the only mandatory argument. Notice that you cannot a bash shell or any of its shell built-ins as the command. So, the following examples will **<u>not work</u>**:
+Both `command` and `jobname` are mandatory arguments. Notice that you cannot a bash shell or any of its shell built-ins as the command. So, the following examples will **<u>not work</u>**:
 
 ```bash
 cronmanager -c "echo 'hello' > somefile"
@@ -79,10 +79,11 @@ For the tool to work, the `/opt/prometheus/exporters/dist/textfile/ `path **<u>m
 Once cronmanager starts a job, it will wait for the specified seconds (using `-t` or the default 3600 seconds). If the cron is still running, cronmanager writes to a file under the exporters path. The file name consists of the job name followed by the `.prom` extension. For example, if you run the command like this `cronmanager -c "some_command some_arguments" -n "myjob"` the following file will be created: `/opt/prometheus/exporters/dist/textfile/myjob.prom`. The contents of the file are as follows:
 
 ```plain
-# TYPE cron_job gauge
-cron_job{"name=cron1","dimension=failed"} 0
-cron_job{"name=cron1","dimension=delayed"} 0
-cron_job{"name=cron1","dimension=duration"} 10
+# HELP cronjob metric generated by cronmanager
+# TYPE cronjob gauge
+cronjob{"name=cron1","dimension=failed"} 0
+cronjob{"name=cron1","dimension=delayed"} 0
+cronjob{"name=cron1","dimension=duration"} 10
 ```
 
 The numbers change to `1` depending on the issue found with the cron job (delayed/failed or both).

main.go

@@ -16,7 +16,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/juju/fslock"
+//	"github.com/juju/fslock"
 )
 
 //isDelayed: Used to signal that the cron job delay was triggered
@@ -74,6 +74,7 @@ func main() {
 			writeToExporter(*jobnamePtr, "duration", strconv.FormatFloat(jobDuration, 'f', 0, 64))
 			// Store last timestamp
 			writeToExporter(*jobnamePtr, "last", fmt.Sprintf("%d", time.Now().Unix()))
+//			writeToExporter(*jobnamePtr, "start", fmt.Sprintf("%d", jobStartTime.Unix()))
 		}
 	}()
 
@@ -140,33 +141,39 @@ func main() {
 	writeToExporter(*jobnamePtr, "last", fmt.Sprintf("%d", time.Now().Unix()))
 }
 
-func getExporterPath() string {
+func getExporterPath(jobName string) string {
 	exporterPath, exists := os.LookupEnv("COLLECTOR_TEXTFILE_PATH")
-	exporterPath = exporterPath + "/crons.prom"
+	exporterPath = exporterPath + "/" + jobName + ".prom"
 	if !exists {
-		exporterPath = "/var/cache/prometheus/crons.prom"
+		exporterPath = "/var/cache/prometheus/" + jobName + ".prom"
 	}
 	return exporterPath
 }
 
 func writeToExporter(jobName string, label string, metric string) {
 	jobNeedle := "cronjob{name=\"" + jobName + "\",dimension=\"" + label + "\"}"
-	typeData := "# TYPE cron_job gauge"
+	// both TYPE and HELP must be the same across all .prom files
+	// otherwise node_exporter textfile won't merge them
+	// see https://github.com/prometheus/node_exporter/issues/1885
+	helpData := "# HELP cronjob metric generated by cronmanager"
+	typeData := "# TYPE cronjob gauge"
 	jobData := jobNeedle + " " + metric
 
-	exporterPath := getExporterPath()
+	exporterPath := getExporterPath(jobName)
 	// Lock filepath to prevent race conditions
-	lock := fslock.New(exporterPath)
-	err := lock.Lock()
-	if err != nil {
-		log.Println("Error locking file " + exporterPath)
-	}
-	defer lock.Unlock()
+	// however, lock also prevents reading
+//	lock := fslock.New(exporterPath+".tmp")
+//	err := lock.Lock()
+//	if err != nil {
+//		log.Println("Error locking file " + exporterPath)
+//	}
+//	defer lock.Unlock()
 
 	input, err := ioutil.ReadFile(exporterPath)
 	if err != nil {
-		// We're not sure why we can't read from the file. Let's try creating it and fail if that didn't work either
-		if _, err := os.Create(exporterPath); err != nil {
+		// We're not sure why we can't read from the file.
+		// Let's try creating it and fail if that didn't work either
+		if _, err := os.Create(exporterPath+".tmp"); err != nil {
 			log.Fatal("Couldn't read or write to the exporter file. Check parent directory permissions")
 		}
 	}
@@ -175,27 +182,30 @@ func writeToExporter(jobName string, label string, metric string) {
 	if re.Match(input) {
 		input = re.ReplaceAll(input, []byte(jobData+"\n"))
 	} else {
-		// If the job is not there then either there is no TYPE header at all and this is the first job
+		// If TYPE line is not there then this is the first run of the job
 		if re := regexp.MustCompile(typeData); !re.Match(input) {
-			// Add the TYPE and the job data
+			// Add HELP, TYPE and the job data
+			input = append(input, helpData+"\n"...)
 			input = append(input, typeData+"\n"...)
 			input = append(input, jobData+"\n"...)
 		} else {
-			// Or there is a TYPE header with one or more other jobs. Just append the job to the TYPE header
+			// Or there is a TYPE header with one or more other jobs. Just append the job to the TYPE headers
 			input = re.ReplaceAll(input, []byte(typeData+"\n"+jobData))
 		}
 	}
-	f, err := os.Create(exporterPath)
+	f, err := os.Create(exporterPath+".tmp")
 	if err != nil {
 		log.Fatal(err)
 	}
 	err = f.Chmod(0644)
 	if err != nil {
 		log.Fatal(err)
 	}
-
-	defer f.Close()
 	if _, err = f.Write(input); err != nil {
 		log.Fatal(err)
 	}
+	f.Close();
+	if err = os.Rename(exporterPath+".tmp", exporterPath); err != nil {
+		log.Fatal(err)
+	}
 }