feat: send logs to userspace + revamp

Update bpf_printk statements to be also sent to userspace so they can be
logged alongside events. All log lines are considered errors and should
be checked (unless running a debug eBPF binary).

Revamps the CO-RE reads that were performed in the eBPF code to use
slimmer types and to use BPF helper macros for readability.

.github/workflows/enterprise-release.yaml

@@ -51,7 +51,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: "^1.20.7"
+          go-version: "^1.20.9"
 
       - name: Build binaries
         run: |

.github/workflows/enterprise.yaml

@@ -37,7 +37,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "^1.20.7"
+          go-version: "^1.20.9"
 
       - name: Echo Go Cache Paths
         id: go-cache-paths

.github/workflows/quality.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: "^1.20.7"
+          go-version: "^1.20.9"
 
       - name: Run make fmt/go
         run: make fmt/go
@@ -75,12 +75,12 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: "^1.20.7"
+          go-version: "^1.20.9"
 
       - name: Install golangci-lint
         run: |
           curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \
-            | sh -s -- -b $(go env GOPATH)/bin v1.53.2
+            | sh -s -- -b $(go env GOPATH)/bin v1.57.2
 
       # Linting needs to be done on each build variation of GOOS.
       - name: Run make lint/go/linux
@@ -100,6 +100,14 @@ jobs:
       - name: Run make lint/c
         run: make lint/c
 
+      - name: Ensure DEBUG is disabled
+        run: |
+          # look for uncommented "#define DEBUG" in bpf/handler.c
+          if grep -q "^#define DEBUG" bpf/handler.c; then
+            echo "DEBUG is enabled in bpf/handler.c"
+            exit 1
+          fi
+
   lint-shellcheck:
     name: lint/shellcheck
     runs-on: ubuntu-20.04

.gitignore

@@ -33,3 +33,5 @@ build
 *.tfplan
 *.lock.hcl
 .terraform/
+
+/exectrace

.golangci.yaml

@@ -20,30 +20,30 @@ linters-settings:
     enabled-checks:
       # - appendAssign
       # - appendCombine
-      - argOrder
+      #- argOrder
       # - assignOp
       # - badCall
-      - badCond
+      #- badCond
       - badLock
       - badRegexp
       - boolExprSimplify
       # - builtinShadow
       - builtinShadowDecl
-      - captLocal
-      - caseOrder
-      - codegenComment
+      #- captLocal
+      #- caseOrder
+      #- codegenComment
       # - commentedOutCode
       - commentedOutImport
-      - commentFormatting
-      - defaultCaseOrder
+      #- commentFormatting
+      #- defaultCaseOrder
       - deferUnlambda
       # - deprecatedComment
       # - docStub
-      - dupArg
-      - dupBranchBody
-      - dupCase
+      #- dupArg
+      #- dupBranchBody
+      #- dupCase
       - dupImport
-      - dupSubExpr
+      #- dupSubExpr
       # - elseif
       - emptyFallthrough
       # - emptyStringTest
@@ -52,56 +52,56 @@ linters-settings:
       # - exitAfterDefer
       # - exposedSyncMutex
       # - filepathJoin
-      - flagDeref
-      - flagName
+      #- flagDeref
+      #- flagName
       - hexLiteral
       # - httpNoBody
       # - hugeParam
       # - ifElseChain
       # - importShadow
       - indexAlloc
       - initClause
-      - mapKey
+      #- mapKey
       - methodExprCall
       # - nestingReduce
-      - newDeref
+      #- newDeref
       - nilValReturn
       # - octalLiteral
-      - offBy1
+      #- offBy1
       # - paramTypeCombine
       # - preferStringWriter
       # - preferWriteByte
       # - ptrToRefParam
       # - rangeExprCopy
       # - rangeValCopy
-      - regexpMust
+      #- regexpMust
       - regexpPattern
       # - regexpSimplify
       #- ruleguard
-      - singleCaseSwitch
-      - sloppyLen
+      #- singleCaseSwitch
+      #- sloppyLen
       # - sloppyReassign
-      - sloppyTypeAssert
+      #- sloppyTypeAssert
       - sortSlice
       - sprintfQuotedString
       - sqlQuery
       # - stringConcatSimplify
       # - stringXbytes
       # - suspiciousSorting
-      - switchTrue
+      #- switchTrue
       - truncateCmp
       - typeAssertChain
       # - typeDefFirst
-      - typeSwitchVar
+      #- typeSwitchVar
       # - typeUnparen
-      - underef
+      #- underef
       # - unlabelStmt
       # - unlambda
       # - unnamedResult
       # - unnecessaryBlock
       # - unnecessaryDefer
       # - unslice
-      - valSwap
+      #- valSwap
       - weakCond
       # - whyNoLint
       # - wrapperFunc
@@ -207,15 +207,17 @@ issues:
       linters:
         - exhaustruct
 
+  exclude-files:
+    - scripts/rules.go
+
+  exclude-dirs:
+    - node_modules
+
   fix: true
   max-issues-per-linter: 0
   max-same-issues: 0
 
 run:
-  skip-dirs:
-    - node_modules
-  skip-files:
-    - scripts/rules.go
   timeout: 10m
 
 # Over time, add more and more linters from

Makefile

@@ -24,12 +24,12 @@ handlers: bpf/handler-bpfeb.o bpf/handler-bpfel.o
 clean: clean-enterprise
 	rm -rf bpf/handler-bpfeb.o bpf/handler-bpfel.o
 
-ci/.clang-image: ci/images/clang-13/Dockerfile
+ci/.clang-image: ci/images/clang-13/Dockerfile ci/scripts/clang_image.sh
 	./ci/scripts/clang_image.sh
 	touch ci/.clang-image
 
 # bpfeb is big endian, bpfel is little endian.
-bpf/handler-bpfeb.o bpf/handler-bpfel.o: bpf/*.h bpf/*.c ci/.clang-image
+bpf/handler-bpfeb.o bpf/handler-bpfel.o: bpf/*.h bpf/*.c ci/.clang-image ci/scripts/build_handler.sh
 	./ci/scripts/build_handler.sh "$(@F)"
 
 .PHONY: fmt

README.md

@@ -113,25 +113,29 @@ func main() {
 You will need the following:
 
 - Docker (the Makefile runs clang within a Docker container for reproducibility)
+- Golang 1.20+
 - `golangci-lint`
 - `prettier`
 - `shellcheck`
 
-Since the eBPF program is packaged as a Go library, you need to compile the
-program and include it in the repo.
+Since the eBPF program is packaged using `go:embed`, you will need to compile
+the program and include it in the repo.
 
 If you change the files in the `bpf` directory, run `make` and ensure that you
 include the `.o` files you changed in your commit (CI will verify that you've
 done this correctly).
 
-## Status: beta
+## Status: stable
 
-This library is ready to use as-is, though it is under active development as we
-modify it to suit the needs of Coder's [enterprise product](https://coder.com).
+This library is ready to use as-is. It has been used in production for years and
+has received minimal maintenance over that time period.
 
-We plan on adding more features and fields that can be read from the API, as
-well as easier-to-use methods for filtering events (currently, you must
-implement additional filtering yourself).
+In April 2024, a system to send logs from the kernel to userspace was added
+which can make discovering potential issues in production/development much
+easier.
+
+The API will likely not be further modified as we have no need for additional
+fields/features. We will continue to maintain the library as needed.
 
 ## See also
 

bpf.go

@@ -53,6 +53,12 @@ func loadBPFObjects() (*bpfObjects, error) {
 	}
 	err = spec.LoadAndAssign(objs, collectionOpts)
 	if err != nil {
+		var ve *ebpf.VerifierError
+		if xerrors.As(err, &ve) {
+			// It's better to use %+v for this as it forces the error to contain
+			// all lines from the verifier log.
+			return nil, xerrors.Errorf("load and assign specs: verifier error: %+v", ve)
+		}
 		return nil, xerrors.Errorf("load and assign specs: %w", err)
 	}
 
@@ -62,6 +68,7 @@ func loadBPFObjects() (*bpfObjects, error) {
 type bpfObjects struct {
 	EnterExecveProg *ebpf.Program `ebpf:"enter_execve"`
 	EventsMap       *ebpf.Map     `ebpf:"events"`
+	LogsMap         *ebpf.Map     `ebpf:"logs"`
 	FiltersMap      *ebpf.Map     `ebpf:"filters"`
 
 	closeLock sync.Mutex
@@ -92,6 +99,18 @@ func (o *bpfObjects) Close() error {
 			merr = multierror.Append(merr, xerrors.Errorf(`close BPF map "events": %w`, err))
 		}
 	}
+	if o.LogsMap != nil {
+		err := o.LogsMap.Close()
+		if err != nil {
+			merr = multierror.Append(merr, xerrors.Errorf(`close BPF map "logs": %w`, err))
+		}
+	}
+	if o.FiltersMap != nil {
+		err := o.FiltersMap.Close()
+		if err != nil {
+			merr = multierror.Append(merr, xerrors.Errorf(`close BPF map "filters": %w`, err))
+		}
+	}
 
 	return merr
 }