From c0366b9baa198f4f6cfd25c8e5f66bd8c1e2a607 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Fri, 17 Jan 2025 11:47:04 +1000 Subject: [PATCH] Bugfix: EVTX: Support multiple messages for the same Event ID Some event ID have multiple messages stored in the message lists - these are generally designed for events which have different number of properties. So for example the message file might contain two messages for the same event id, one with 1 expansion and one with 2 expansions. Then the application might emit an event to the log file with 2 properties or only 1 property of the same event id. This pr stores both the messages and the number of expasions in the message set and is able to select the most appropriate one for each message - we aim to maximize the number of expasions available in the message string. --- .github/workflows/go.yml | 3 +-- .github/workflows/musl.yaml | 6 ++++-- go.mod | 6 +++--- go.sum | 11 ++++++----- vql/parsers/json.go | 6 ++++++ 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 22d197cf8ba..33096f0f54e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -30,7 +30,7 @@ jobs: go get -v -t -d ./... sudo apt-get update sudo apt-get upgrade - sudo apt-get install mingw-w64-x86-64-dev gcc-mingw-w64-x86-64 gcc-mingw-w64 gcc-aarch64-linux-gnu + sudo apt-get install mingw-w64-x86-64-dev gcc-mingw-w64-x86-64 gcc-mingw-w64 - name: Use Node.js uses: actions/setup-node@v4 @@ -52,7 +52,6 @@ jobs: export PATH=$PATH:~/go/bin/ go run make.go -v UpdateDependentTools go run make.go -v Linux - go run make.go -v LinuxArm64 go run make.go -v Windows go run make.go -v Windowsx86 go run make.go -v DarwinBase diff --git a/.github/workflows/musl.yaml b/.github/workflows/musl.yaml index 6bb9e3c5781..7eae4f679b8 100644 --- a/.github/workflows/musl.yaml +++ b/.github/workflows/musl.yaml @@ -25,7 +25,7 @@ jobs: run: | go get -v -t -d ./... sudo apt-get update - sudo apt-get install -y zip build-essential pkg-config libssl-dev + sudo apt-get install -y zip build-essential pkg-config libssl-dev gcc-aarch64-linux-gnu - name: Install Musl run: | @@ -54,7 +54,9 @@ jobs: export PATH=$PATH:~/go/bin/:/usr/local/musl/bin go run make.go -v UpdateDependentTools go run make.go -v LinuxMusl - go run make.go -v LinuxMuslDebug + # go run make.go -v LinuxMuslDebug + go run make.go -v Linux + go run make.go -v LinuxArm64 - name: StoreBinaries uses: actions/upload-artifact@v4 diff --git a/go.mod b/go.mod index 427dc8c046d..ad45ac4dd5e 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/magefile/mage v1.15.0 github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-pointer v0.0.0-20180825124634-49522c3f3791 - github.com/mattn/go-sqlite3 v1.14.22 + github.com/mattn/go-sqlite3 v1.14.24 github.com/microcosm-cc/bluemonday v1.0.23 github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/panicwrap v1.0.0 @@ -74,7 +74,7 @@ require ( golang.org/x/crypto v0.31.0 golang.org/x/mod v0.21.0 golang.org/x/net v0.33.0 - golang.org/x/sys v0.28.0 + golang.org/x/sys v0.29.0 golang.org/x/text v0.21.0 golang.org/x/time v0.5.0 google.golang.org/api v0.169.0 @@ -86,7 +86,7 @@ require ( gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/sourcemap.v1 v1.0.5 // indirect howett.net/plist v1.0.0 - www.velocidex.com/golang/evtx v0.2.1-0.20240730174545-3e4ff3d96433 + www.velocidex.com/golang/evtx v0.2.1-0.20250117005955-e5cd153ed377 www.velocidex.com/golang/go-ese v0.2.1-0.20240919031214-2aa005106db2 www.velocidex.com/golang/go-ntfs v0.2.1-0.20241123135758-e6f7e1f1c474 www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b diff --git a/go.sum b/go.sum index bf9e45b8fd6..0113286c8c2 100644 --- a/go.sum +++ b/go.sum @@ -510,8 +510,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= @@ -826,8 +826,9 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -960,8 +961,8 @@ www.velocidex.com/golang/binparsergen v0.1.0/go.mod h1:UC43Ecj0mjsidlClTYZ3H4dXd www.velocidex.com/golang/binparsergen v0.1.1-0.20220107080050-ae6122c5ed14/go.mod h1:Q/J/huOyH6IlY2aShigY1CnZnw5EO0+FZJgnGEBrT5Q= www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e h1:uf1AsYiIzUMJMIdFsVdrIw/BjrGzZbrsnz9xmeZmlYU= www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e/go.mod h1:jk+uZGukrJZWgnNH6q9tJLUnbugHEDPCQdIOmBBMXY4= -www.velocidex.com/golang/evtx v0.2.1-0.20240730174545-3e4ff3d96433 h1:qrRlDit2WJgfGA4xjNq9/xdFJQGkrXfe1BuJRkZ41jA= -www.velocidex.com/golang/evtx v0.2.1-0.20240730174545-3e4ff3d96433/go.mod h1:z0QWgpVDct1l+cHNq64vrSWdFuY6/BgrW2f/Qrc6oK4= +www.velocidex.com/golang/evtx v0.2.1-0.20250117005955-e5cd153ed377 h1:dJn+CMhWi5mi2VSdtBjWXLhNaGyVZKdIYTTM4RJGfbU= +www.velocidex.com/golang/evtx v0.2.1-0.20250117005955-e5cd153ed377/go.mod h1:JDMB7j3uBFgww0+PzsQUGvnOywFEHkbynzAPyNvhiAg= www.velocidex.com/golang/go-ese v0.2.1-0.20240919031214-2aa005106db2 h1:f7nj4NsyeMSrwiFd9XO/VfsZYt6o6FH1KJmmqlBZDgM= www.velocidex.com/golang/go-ese v0.2.1-0.20240919031214-2aa005106db2/go.mod h1:YKxCStqE15c6F/P81oCG0Y5oelDBah2hCdO6P+VPUIQ= www.velocidex.com/golang/go-ntfs v0.2.1-0.20241123135758-e6f7e1f1c474 h1:iaV0M55ZTdVU9nIqcHkQKwUfQOOoswC0eBZsKvlPN/0= diff --git a/vql/parsers/json.go b/vql/parsers/json.go index 03432401ff8..49eb3ea1595 100644 --- a/vql/parsers/json.go +++ b/vql/parsers/json.go @@ -104,7 +104,13 @@ func (self ParseJsonArray) Call( return &vfilter.Null{} } + arg.Data = strings.TrimSpace(arg.Data) + result_array := []json.RawMessage{} + if arg.Data == "" { + return result_array + } + err = json.Unmarshal([]byte(arg.Data), &result_array) if err != nil { scope.Log("parse_json_array: %v", err)