v1.1.0-beta

.github/workflows/co-re.yaml

@@ -7,7 +7,7 @@ on:
     branches: [main]
 
 env:
-  GO_VERSION: "1.17"
+  GO_VERSION: "1.18"
 
 jobs:
   CO-RE:

.github/workflows/non-co-re-daily.yaml

@@ -5,7 +5,7 @@ on:
     - cron: "0 0 * * *"
 
 env:
-  GO_VERSION: "1.17"
+  GO_VERSION: "1.18"
 
 jobs:
   non-CO-RE-ubuntu-latest:

.github/workflows/release-driver.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: go env
         uses: actions/setup-go@v2
         with:
-          go-version: '1.17'
+          go-version: '1.18'
 
       - name: llvm env
         run: |

.gitignore

@@ -29,6 +29,7 @@ cert/
 # ignore binary
 plugin/ebpfdriver/ebpfdriver
 plugin/collector/collector
+plugin/ncp/ncp
 
 # libbpf test
 plugin/ebpfdriver/kern/headers/libbpf

SDK/logger/logger.go

@@ -14,17 +14,6 @@ import (
 	"github.com/chriskaliX/SDK/transport/client"
 )
 
-var _ ILogger = (*zap.SugaredLogger)(nil)
-
-type ILogger interface {
-	Debug(args ...interface{})
-	Info(args ...interface{})
-	Warn(args ...interface{})
-	Error(args ...interface{})
-	Fatal(args ...interface{})
-	Panic(args ...interface{})
-}
-
 type Config struct {
 	// SDK fields
 	Client *client.Client
@@ -38,7 +27,7 @@ type Config struct {
 	RemoteLevel zapcore.LevelEnabler
 }
 
-func New(config *Config) *zap.SugaredLogger {
+func New(config *Config) *zap.Logger {
 	var l *zap.Logger
 	remoteConfig := zap.NewProductionEncoderConfig()
 	remoteConfig.CallerKey = "source"
@@ -67,5 +56,5 @@ func New(config *Config) *zap.SugaredLogger {
 	)
 	l = zap.New(core, zap.AddCaller())
 	zap.ReplaceGlobals(l)
-	return zap.S()
+	return l
 }

SDK/sandbox.go

@@ -19,6 +19,7 @@ import (
 	"github.com/chriskaliX/SDK/transport/protocol"
 	"github.com/chriskaliX/SDK/util/hash"
 	"github.com/nightlyone/lockfile"
+	"go.uber.org/zap"
 )
 
 var _ ISandbox = (*Sandbox)(nil)
@@ -45,7 +46,7 @@ type ISandbox interface {
 type Sandbox struct {
 	// required fields
 	Clock  clock.IClock
-	Logger logger.ILogger
+	Logger *zap.Logger
 	Client *client.Client
 	name   string
 	// Optional fields
@@ -82,11 +83,6 @@ func (s *Sandbox) Init(sconfig *SandboxConfig) error {
 	sconfig.LogConfig.Clock = s.Clock
 	sconfig.LogConfig.Client = s.Client
 	s.Logger = logger.New(sconfig.LogConfig)
-	// lockfile for plugin
-	// if err := s.Lockfile(); err != nil {
-	// 	zap.S().Errorf("init failed with lockfile %s", err.Error())
-	// 	return err
-	// }
 	defer s.Logger.Info(fmt.Sprintf("sandbox %s init", s.name))
 	// Optional fields initialization
 	if sconfig.Hash {
@@ -106,11 +102,12 @@ func (s *Sandbox) Run(mfunc func(ISandbox) error) (err error) {
 	defer s.Logger.Info(fmt.Sprintf("%s is exited", s.name))
 	s.Logger.Info(fmt.Sprintf("%s run is called", s.name))
 	if err = mfunc(s); err != nil {
+		zap.S().Error("sandbox main func failed, %s", err.Error())
 		return err
 	}
 	s.Logger.Info(fmt.Sprintf("%s is running", s.name))
 	// os.Interrupt for command line
-	signal.Notify(s.sigs, syscall.SIGTERM)
+	signal.Notify(s.sigs, syscall.SIGTERM, syscall.SIGTERM, os.Interrupt)
 	for {
 		select {
 		case sig := <-s.sigs:
@@ -224,7 +221,7 @@ func (s *Sandbox) recvTask() {
 		default:
 			task, err := s.Client.ReceiveTask()
 			if err != nil {
-				s.Logger.Error(err)
+				s.Logger.Error(err.Error())
 				time.Sleep(5 * time.Second)
 				continue
 			}

SDK/transport/pool/pool.go

@@ -14,26 +14,20 @@ var BufferPool = NewPool()
 type Pool struct {
 	p1 *sync.Pool
 	p2 *sync.Pool
+	p3 *sync.Pool
 }
 
 // For plugin ebpfdriver as an example, most of the data sizes are within
 // 2048, drop size over 4096 as default
 func NewPool() Pool {
 	return Pool{
-		p1: &sync.Pool{
-			New: func() interface{} {
-				return make([]byte, 2<<10)
-			},
-		},
-		p2: &sync.Pool{
-			New: func() interface{} {
-				return make([]byte, 4<<10)
-			},
-		},
+		p1: &sync.Pool{New: func() interface{} { return make([]byte, 2<<10) }},
+		p2: &sync.Pool{New: func() interface{} { return make([]byte, 4<<10) }},
+		p3: &sync.Pool{New: func() interface{} { return make([]byte, 8<<10) }},
 	}
 }
 
-func (p Pool) Get(size int64) []byte {
+func (p *Pool) Get(size int64) []byte {
 	if size <= 2<<10 {
 		return p.p1.Get().([]byte)
 	} else if size <= 4<<10 {
@@ -42,13 +36,18 @@ func (p Pool) Get(size int64) []byte {
 	return make([]byte, size)
 }
 
-func (p Pool) Put(b []byte) {
+func (p *Pool) Put(b []byte) {
 	if len(b) <= 2<<10 {
 		p.p1.Put(b)
 		return
 	}
 	if len(b) <= 4<<10 {
 		p.p2.Put(b)
+		return
+	}
+	if len(b) <= 8<<10 {
+		p.p3.Put(b)
+		return
 	}
 	// deprecate []byte over 4096, let GC collects them
 }

SDK/util/connection/connection.go

@@ -24,7 +24,7 @@ type INetRetry interface {
 	GetMaxDelay() uint
 }
 
-func IRetry(netRetry INetRetry, ctx context.Context) (err error) {
+func IRetry(ctx context.Context, netRetry INetRetry) (err error) {
 	var (
 		maxRetries   uint
 		maxDelay     uint
@@ -45,7 +45,7 @@ func IRetry(netRetry INetRetry, ctx context.Context) (err error) {
 			return
 		default:
 			if maxRetries > 0 && retries >= maxRetries {
-				err = fmt.Errorf("Abandon %s after %d retries.", netRetry.String(), retries)
+				err = fmt.Errorf("abandon %s after %d retries.", netRetry.String(), retries)
 				zap.S().Error(err)
 				return err
 			}

SDK/util/download.go

@@ -6,39 +6,37 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"time"
 
 	"go.uber.org/zap"
 )
 
-func CheckSignature(dst string, sign string) (err error) {
-	var (
-		f         *os.File
-		signBytes []byte
-	)
-	if f, err = os.Open(dst); err != nil {
-		return
+var downloadTimeout = 10 * time.Minute
+
+func CheckSignature(dst string, sign string) error {
+	f, err := os.Open(dst)
+	if err != nil {
+		return err
 	}
 	defer f.Close()
-	if signBytes, err = hex.DecodeString(sign); err != nil {
-		return
+	signBytes, err := hex.DecodeString(sign)
+	if err != nil {
+		return err
 	}
 	hasher := sha256.New()
-	//  https://pandaychen.github.io/2020/01/01/MAGIC-GO-IO-PACKAGE/
 	if _, err = io.Copy(hasher, f); err != nil {
-		return
+		return err
 	}
 	if !bytes.Equal(hasher.Sum(nil), signBytes) {
 		err = errors.New("signature doesn't match")
-		return
+		return err
 	}
-	// make it executable
 	f.Chmod(0o0700)
-	return
+	return nil
 }
 
 // Download function get the plugin executable file from urls
@@ -48,50 +46,63 @@ func CheckSignature(dst string, sign string) (err error) {
 // memory is always needed
 func Download(ctx context.Context, dst string, sha256sum string, urls []string, suffix string) (err error) {
 	var checksum []byte
+	// check wheater this already exist
 	if checksum, err = hex.DecodeString(sha256sum); err != nil {
 		return
 	}
 	hasher := sha256.New()
-	// extra work, but to simplify
 	if err = CheckSignature(dst, sha256sum); err == nil {
 		return
 	}
-	// in elkeid, `defer` in loop... emmm, not a best practice I think, but nothing wrong
+	// In Elkeid v1.9.1, only Timeout is different from DefaultTransport.
+	// Before v1.9.1 the timeout was controlled by subctx, now it is
+	// controlled by client itself.
+	client := &http.Client{
+		Transport: http.DefaultTransport,
+		Timeout:   downloadTimeout,
+	}
 	for _, rawurl := range urls {
 		var req *http.Request
 		var resp *http.Response
-		var buf []byte
-		subctx, cancel := context.WithTimeout(ctx, time.Minute*3)
+		subctx, cancel := context.WithCancel(ctx)
 		defer cancel()
 		if req, err = http.NewRequestWithContext(subctx, "GET", rawurl, nil); err != nil {
 			continue
 		}
-		if resp, err = http.DefaultClient.Do(req); err != nil {
+		if resp, err = client.Do(req); err != nil {
 			continue
 		}
 		if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
 			err = errors.New("http error: " + resp.Status)
 			continue
 		}
-		defer resp.Body.Close()
-		if buf, err = ioutil.ReadAll(resp.Body); err != nil {
-			continue
-		}
+		// Set the limitation of download, but I think it would be alright
+		// since the time limitation is also set. This configration helps
+		// avoid download too much to fill up the disk even within 10 mins.
+		resp.Body = http.MaxBytesReader(nil, resp.Body, 512*1024*1024)
 		hasher.Reset()
-		hasher.Write(buf)
-		if !bytes.Equal(hasher.Sum(nil), checksum) {
-			err = errors.New("checksum doesn't match")
-			continue
-		}
-		br := bytes.NewBuffer(buf)
-		zap.S().Info("start to decompress to ", dst)
+		// Before Elkeid v1.9.1, ioutil.ReadAll is used and it may
+		// lead memory grows rapidly since it reads everything into
+		// memeory, using io.TeeReader to both read from Reader and
+		// calculate the hash
+		// Also the decompress should be updated
+		r := io.TeeReader(resp.Body, hasher)
 		switch suffix {
 		case "tar.gz":
-			err = DecompressTarGz(dst, br)
+			err = DecompressTarGz(dst, r)
 		default:
-			err = DecompressDefault(dst, br)
+			err = DecompressDefault(dst, r)
+		}
+		resp.Body.Close()
+		if err != nil {
+			continue
+		}
+		if c := hex.EncodeToString(hasher.Sum(nil)); c != sha256sum {
+			err = fmt.Errorf("checksum doesn't match: %s vs %s", checksum, sha256sum)
+			zap.S().Error(err)
+		} else {
+			break
 		}
-		break
 	}
 	return
 }

agent/Makefile

@@ -1,5 +1,5 @@
 CMD_GO ?= go
-VERSION ?= 1.0.1
+VERSION ?= v1.0.2
 
 # colors
 INFO_COLOR = \033[34m[*]\033[0m
@@ -14,4 +14,4 @@ pre_show:
 
 .PHONY: build_agent
 build_agent:
-	$(CMD_GO) build -ldflags "-X agent/agent.Version='$(VERSION)'" -o hades-agent
+	$(CMD_GO) build -ldflags "-X agent/agent.Version=$(VERSION)" -o hades-agent

agent/agent/agent.go

@@ -2,23 +2,31 @@ package agent
 
 import (
 	"context"
+	"os"
 )
 
 const (
 	Product = "hades-agent"
 	EnvName = "SPECIFIED_AGENT_ID_HADES"
-	Version = ""
 )
 
-// The only instance of the agent
-var Instance = New()
+var (
+	Version         string // compile time added
+	Context, Cancel = context.WithCancel(context.Background())
+	Workdir, _      = os.Getwd()
+	ID              string
+)
 
-type Agent struct {
-	ID      string
-	Workdir string
-	Version string
-	Context context.Context
-	Cancel  context.CancelFunc
-	Product string
-	OS      string
+func init() {
+	var err error
+	if Workdir, err = os.Getwd(); err != nil {
+		Workdir = HADES_PIDPATH
+	}
+	// ID init
+	var ok bool
+	if ID, ok = os.LookupEnv(EnvName); ok {
+		return
+	}
+	genUUID()
+	os.WriteFile("machine-id", []byte(ID), 0600)
 }