WIP: Add more testing + memory stats from cgroups

.gitignore

@@ -1,2 +1,4 @@
 /systemd_exporter
 /bin/golangci-lint
+.idea
+coverage.txt

.golangci.yml

@@ -0,0 +1,5 @@
+issues:
+  exclude:
+    - "not declared by package utf8"
+    - "unicode/utf8/utf8.go"
+

.travis.yml

@@ -1,4 +1,36 @@
-language: go
+# TODO Use CodeCov 'Flags' to separate coverage for integration tests and unit 
+# tests. Near 100% on integration is expected - these tests cast a wide net to 
+# catch many issues (but do not easily tell you where the issue is). Near 100%
+# on unit tests is a feat of heroism - these tests identify a specific code 
+# chunk with an issue. We mainly care about 100% on unit tests, but 100% on 
+# integration is an easy win and a nice to have 
+#
+# See https://docs.codecov.io/docs/flags
+# Use go get github.com/stristr/go-acc && go-acc ./...
+# Or use coverpkg=github.com/povilasv/systemd_exporter,github.com/povilasv/systemd_exporter/systemd
 
+# This defines the script for us automatically. By default it installs 
+# to requested go version then runs make
+language: go
 go:
   - "1.x"
+
+before_script: systemd --version
+os: linux
+
+go:
+  - 1.x 
+
+before_script: systemd --version && systemctl list-units
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
+
+jobs:
+  include:
+  - dist: xenial
+    name: xenial-229
+  - dist: bionic
+    name: bionic-237
+
+

CHANGELOG.md

@@ -2,20 +2,18 @@
 
 ### **Breaking changes**
 
-* `systemd_unit_state` label `type` has new meaning
-   Now shows Unit type (`service`, `scope`, etc), not Service Unit types (`simple`, `forking`, etc)
-   or mount unit types(`aufs`,`ext3`, etc). Service and mount types have been moved to `systemd_unit_info` 
+* `systemd_unit_state` label `type` has new meaning. Previously `type` contained service unit type (`simple`, `forking`, etc) or mount unit types (`aufs`, `ext3`, etc). Now `systemd_unit_state{type}` contains overall unit type (`service`, `scope`, etc) to allow easy PromQL group by clauses. Service and mount types have been moved to `systemd_unit_info` 
 
 ### Changes
 
 - [FEATURE] Read unit CPU usage from cgroup. Added `systemd_unit_cpu_seconds_total` metric. **Note** - Untested on unified hierarchy
 - [FEATURE] Add `systemd_unit_info` with metainformation about units incl. subtype specific info
 - [ENHANCEMENT] Added `type` label to all metrics named `systemd_unit-*` to support PromQL grouping
-* [ENHANCEMENT] `systemd_unit_state` works for all unit types, not just service and mount units
-* [ENHANCEMENT] Scrapes are approx 80% faster. If needed, set GOMAXPROCS to limit max concurrency
-* [CHANGE] Start tracking metric cardinality in readme
-* [CHANGE] Expanded default set of unit types monitored. Only device unit types are not enabled by default
-* [BUGFIX] `timer_last_trigger_seconds` metric is now exported as expected for all timers
+- [ENHANCEMENT] `systemd_unit_state` works for all unit types, not just service and mount units
+- [ENHANCEMENT] Scrapes are approx 80% faster. If needed, set GOMAXPROCS to limit max concurrency
+- [CHANGE] Start tracking metric cardinality in readme
+- [CHANGE] Expanded default set of unit types monitored. Only device unit types are not enabled by default
+- [BUGFIX] `timer_last_trigger_seconds` metric is now exported as expected for all timers
 
 ## 0.2.0 / 2019-03-20
 

Makefile

@@ -5,7 +5,7 @@ BRANCH := $(shell git branch | grep \* | cut -d ' ' -f2)
 
 LINT_FLAGS := run --deadline=120s
 LINTER := ./bin/golangci-lint
-TESTFLAGS := -v -cover
+TEST_FLAGS := -v -cover -race -coverprofile=coverage.txt -covermode=atomic
 
 GO111MODULE := on
 all: $(LINTER) deps test lint build
@@ -23,7 +23,12 @@ deps:
 
 .PHONY: test
 test:
-	go test $(TESTFLAGS) ./...
+ifdef TRAVIS
+	sudo sh -c 'echo DefaultCPUAccounting=yes >> /etc/systemd/system.conf'  
+	sudo sh -c 'echo DefaultMemoryAccounting=yes >> /etc/systemd/system.conf'
+	sudo systemctl daemon-reload
+endif 
+	go test $(TEST_FLAGS) ./...
 
 .PHONY: build
 build: deps

README.md

@@ -79,19 +79,21 @@ Note that a number of unit types are filtered by default
 | ----------------------------------------- | ----------- | -------- | ------------------------------------------------------------------ |
 | systemd_exporter_build_info               | Gauge       | UNSTABLE | 1 per systemd-exporter                                             |
 | systemd_unit_info                         | Gauge       | UNSTABLE | 1 per service + 1 per mount                                        |
-| systemd_unit_cpu_seconds_total            | Gauge       | UNSTABLE | 2 per mount/scope/slice/socket/swap {mode="system/user"}           |
+| systemd_unit_cpu_seconds_total            | Counter     | UNSTABLE | <sup>1</sup>2 per mount/scope/slice/socket/swap {mode="system/user"}|
 | systemd_unit_state                        | Gauge       | UNSTABLE | 5 per unit {state="activating/active/deactivating/failed/inactive} |
 | systemd_unit_tasks_current                | Gauge       | UNSTABLE | 1 per service                                                      |
 | systemd_unit_tasks_max                    | Gauge       | UNSTABLE | 1 per service                                                      |
 | systemd_unit_start_time_seconds           | Gauge       | UNSTABLE | 1 per service                                                      |
-| systemd_service_restart_total             | Gauge       | UNSTABLE | 1 per service                                                      |
+| systemd_service_restart_total             | Counter     | UNSTABLE | 1 per service                                                      |
 | systemd_socket_accepted_connections_total | Counter     | UNSTABLE | 1 per socket                                                       |
 | systemd_socket_current_connections        | Gauge       | UNSTABLE | 1 per socket                                                       |
-| systemd_socket_refused_connections_total  | Gauge       | UNSTABLE | 1 per socket                                                       |
+| systemd_socket_refused_connections_total  | Counter     | UNSTABLE | 1 per socket. Requires systemd>239                                 |
 | systemd_timer_last_trigger_seconds        | Gauge       | UNSTABLE | 1 per timer                                                        |
 | systemd_process_resident_memory_bytes     | Gauge       | UNSTABLE | 1 per service                                                      |
 | systemd_process_virtual_memory_bytes      | Gauge       | UNSTABLE | 1 per service                                                      |
 | systemd_process_virtual_memory_max_bytes  | Gauge       | UNSTABLE | 1 per service                                                      |
 | systemd_process_open_fds                  | Gauge       | UNSTABLE | 1 per service                                                      |
 | systemd_process_max_fds                   | Gauge       | UNSTABLE | 1 per service                                                      |
 | systemd_process_cpu_seconds_total         | Counter     | UNSTABLE | 1 per service                                                      |
+
+<sup>1</sup>Only present for units which have systemd `CPUAccounting` enabled

cgroup/Readme.md

@@ -0,0 +1,49 @@
+
+This package provides functions to retrieve control group metrics from the pseudo-filesystem `/sys/cgroup/`.
+
+**WARNING:** This package is a work in progress. Its API may still break in backwards-incompatible ways without warnings. Use it at your own risk.
+
+The Linux kernel supports two APIs for userspace to interact with control groups, the v1 API and the v2 API. See 
+[this LWN Article](https://lwn.net/Articles/679786/) or 
+[this kernel documentation](https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#deprecated-v1-core-features) 
+for background on the two APIs. This package will interact with both v1 and v2 APIs.
+
+
+### Focus on Systemd
+
+This package is initially focused on reading metrics for systemd units. Therefore, 
+the following systemd documentation is relevant. 
+
+#### Systemd cgroup mount mode
+
+The kernel can mount the cgroupfs in any manner it chooses. However, anyone wanting to use that cgroupfs must know 
+where/how it is mounted. When there was only one cgroup API, it was always mounted at `/sys/fs/cgroup`. With the 
+transition from v1 to v2, the mounting approach differs per-distro, with some mounting only v2, some mounting only 
+v1(all hierarchies), and some mounting a combination. For simplicity, this package initially focuses on the three 
+mount "modes" supported by systemd: 
+
+via [systemd.io](https://systemd.io/CGROUP_DELEGATION/#three-different-tree-setups-)
+
+1. Unified — this is the simplest mode, and exposes a pure cgroup v2 logic
+2. Legacy — this is the traditional cgroup v1 mode. In this mode the various controllers each get their own cgroup 
+   file system mounted to `/sys/fs/cgroup/<controller>/`
+3. Hybrid — this is a hybrid between the unified and legacy mode. It’s set up mostly like legacy
+
+#### Systemd Supported Controllers
+
+The initial target controllers this package aims to read from are the controllers supported by systemd. Reading from 
+other controllers may be supported in the future. Systemd guarantees that all v1 hierarchies are kept in sync. 
+
+Via [systemd.io](https://systemd.io/CGROUP_DELEGATION/#controller-support):
+
+Systemd supports a number of controllers (but not all). Specifically, supported are:
+
+on cgroup v1: cpu, cpuacct, blkio, memory, devices, pids
+on cgroup v2: cpu, io, memory, pids
+
+It is our intention to natively support all cgroup v2 controllers as they are added 
+to the kernel. However, regarding cgroup v1: at this point we will not add support 
+for any other controllers anymore. This means systemd currently does not and will 
+never manage the following controllers on cgroup v1: freezer, cpuset, net_cls, 
+perf_event, net_prio, hugetlb
+

cgroup/cgroup.go

@@ -0,0 +1,178 @@
+package cgroup
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+	"github.com/prometheus/common/log"
+	"golang.org/x/sys/unix"
+	"os"
+	"path/filepath"
+)
+
+// FS is the pseudo-filesystem cgroupfs, which provides an interface to
+// kernel data structures
+type FS struct {
+	mountPoint string
+
+	// WARNING: We only read this data once at process start, systemd updates
+	// may require restarting systemd-exporter
+	cgroupUnified cgUnifiedMountMode
+}
+
+// DefaultMountPoint is the common mount point of the cgroupfs filesystem
+const DefaultMountPoint = "/sys/fs/cgroup"
+
+// NewDefaultFS returns a new cgroup FS mounted under the default mountPoint.
+// It will error if cgroup hierarchies are not laid out in a manner understood
+// by systemd.
+func NewDefaultFS() (FS, error) {
+
+	mode, err := cgUnifiedCached()
+	if err != nil || mode == unifModeUnknown {
+		return FS{}, fmt.Errorf("could not determine cgroupfs mount mode: %s", err)
+	}
+
+	return newFS(DefaultMountPoint, mode)
+}
+
+// newFS returns a new cgroup FS mounted under the given mountPoint. It does not check
+// the provided mount mode
+func newFS(mountPoint string, mountMode cgUnifiedMountMode) (FS, error) {
+	info, err := os.Stat(mountPoint)
+	if err != nil {
+		return FS{}, fmt.Errorf("could not read %s: %s", mountPoint, err)
+	}
+	if !info.IsDir() {
+		return FS{}, fmt.Errorf("mount point %s is not a directory", mountPoint)
+	}
+	return FS{mountPoint, mountMode}, nil
+}
+
+// path appends the given path elements to the filesystem path, adding separators
+// as necessary.
+func (fs FS) path(p ...string) string {
+	return filepath.Join(append([]string{string(fs.mountPoint)}, p...)...)
+}
+
+// cgUnifiedMountMode constant values describe how cgroup filesystems (aka hierarchies) are
+// mounted underneath /sys/fs/cgroup. In cgroups-v1 there are many mounts,
+// one per controller (cpu, blkio, etc) and one for systemd itself. In
+// cgroups-v2 there is only one mount managed entirely by systemd and
+// internally exposing all controller syscalls. As kernel+distros migrate towards
+// cgroups-v2, systemd has a hybrid mode where it mounts v2 and uses
+// that for process management but also mounts all the v1 filesystem
+// hierarchies and uses them for resource accounting and control
+type cgUnifiedMountMode int8
+
+const (
+	// unifModeUnknown indicates that we do not know if/how any
+	// cgroup filesystems are mounted underneath /sys/fs/cgroup
+	unifModeUnknown cgUnifiedMountMode = iota
+	// unifModeNone indicates that both systemd and the controllers
+	// are using v1 legacy mounts and there is no usage of the v2
+	// unified hierarchy. a.k.a "legacy hierarchy"
+	unifModeNone cgUnifiedMountMode = iota
+	// unifModeSystemd indicates that systemd is using a v2 unified
+	// hierarcy for organizing processes into control groups, but all
+	// controller interaction is using v1 per-controller hierarchies.
+	// a.k.a. "hybrid hierarchy"
+	unifModeSystemd cgUnifiedMountMode = iota
+	// unifModeAll indicates that v2 API is in full usage and there
+	// are no v1 hierarchies exported. Programs (mainly container orchestrators
+	// such as docker,runc,etc) that rely on v1 APIs will be broken.
+	// a.k.a. "unified hierarchy"
+	unifModeAll cgUnifiedMountMode = iota
+)
+func (c cgUnifiedMountMode) String() string {
+	return [...]string{"unknown", "none", "systemd", "all"}[c]
+}
+
+
+// Values copied from https://github.com/torvalds/linux/blob/master/include/uapi/linux/magic.h
+const (
+	tmpFsMagic        = 0x01021994
+	cgroupSuperMagic  = 0x27e0eb
+	cgroup2SuperMagic = 0x63677270
+)
+
+// cgUnifiedCached checks the filesystem types mounted under /sys/fs/cgroup to determine
+// which systemd layout (legacy/hybrid/unified) is in use.
+// We do not bother to track unified_systemd_v232 as our usage does not
+// depend on reading the systemd hierarchy directly, we only focus on reading
+// the controllers. If you care if /sys/fs/cgroup/systemd is v1 or v2 you need
+// to track this
+// WARNING: We cache this data once at process start. Systemd updates
+// may require restarting systemd-exporter
+// Equivalent to systemd cgroup-util.c#cg_unified_cached
+var statfsFunc = unix.Statfs
+func cgUnifiedCached() (cgUnifiedMountMode, error) {
+	// if cgroupUnified != unifModeUnknown {
+	// 	return cgroupUnified, nil
+	// }
+
+	var fs unix.Statfs_t
+	err := statfsFunc("/sys/fs/cgroup/", &fs)
+	if err != nil {
+		return unifModeUnknown, errors.Wrapf(err, "failed statfs(/sys/fs/cgroup/)")
+	}
+
+	switch fs.Type {
+	case cgroup2SuperMagic:
+		log.Debugf("Found cgroup2 on /sys/fs/cgroup/, full unified hierarchy")
+		return unifModeAll, nil
+	case tmpFsMagic:
+		err := statfsFunc("/sys/fs/cgroup/unified/", &fs)
+
+		// Ignore err, we expect path to be missing on v232
+		if err == nil && fs.Type == cgroup2SuperMagic {
+			log.Debugf("Found cgroup2 on /sys/fs/cgroup/unified, unified hierarchy for systemd controller")
+			return unifModeSystemd, nil
+		}
+
+		err = statfsFunc("/sys/fs/cgroup/systemd/", &fs)
+		if err != nil {
+			return unifModeUnknown, errors.Wrapf(err, "failed statfs(/sys/fs/cgroup/systemd)")
+		}
+
+		switch fs.Type {
+		case cgroup2SuperMagic:
+			log.Debugf("Found cgroup2 on /sys/fs/cgroup/systemd, unified hierarchy for systemd controller (v232 variant)")
+			return unifModeSystemd, nil
+		case cgroupSuperMagic:
+			log.Debugf("Found cgroup on /sys/fs/cgroup/systemd, legacy hierarchy")
+			return unifModeNone, nil
+		default:
+			return unifModeUnknown, errors.Errorf("unknown magic number %x for fstype returned by statfs(/sys/fs/cgroup/systemd)", fs.Type)
+		}
+
+	default:
+		return unifModeUnknown, errors.Errorf("unknown magic number %x for fstype returned by statfs(/sys/fs/cgroup)", fs.Type)
+	}
+}
+
+// cgGetPath returns the absolute path for a specific file in a specific controller
+// in the specific cgroup denoted by the passed subpath.
+// Input examples: ("cpu", "/system.slice", "cpuacct.usage_all")
+func (fs FS) cgGetPath(controller string, subpath string, suffix string) (string, error) {
+	// relevant systemd source code in cgroup-util.[h|c] specifically cg_get_path
+	//  2. Joins controller name with base path
+
+	if fs.cgroupUnified == unifModeUnknown {
+		return "", errors.Errorf("Cannot determine path with unknown mounting hierarchy")
+	}
+
+	// TODO Ensure controller name is valid
+	// TODO Convert controller name into guaranteed valid directory name
+	dn := controller
+
+	joined := ""
+	switch fs.cgroupUnified {
+	case unifModeNone, unifModeSystemd:
+		joined = fs.path(dn, subpath, suffix)
+	case unifModeAll:
+		joined = fs.path(subpath, suffix)
+	default:
+		return "", errors.Errorf("unknown cgroup mount mode (e.g. unified mode) %d", fs.cgroupUnified)
+	}
+	return joined, nil
+}