From 2d1e27046a28361c9a4b189abbe01391ba09d4e5 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Fri, 18 Oct 2024 08:54:54 +1000 Subject: [PATCH] Bugfix: Disable support for FlowStatsRequest in pool client (#3840) The responses were not forwarded causing the server to constantly query about the state of the flows. Also added debug view into directly connected clients. Added tags to timeouts so we can identify the cause of timeout cancellations better. --- api/api.go | 11 +++- api/builder.go | 20 +++--- api/query.go | 31 ++++----- bin/query.go | 6 +- executor/pool.go | 8 +++ flows/client_flow_runner.go | 41 ++++++++++-- go.mod | 28 ++++---- go.sum | 66 ++++++++----------- .../src/components/core/mode-yaml.jsx | 5 +- .../src/components/core/table.jsx | 4 ++ .../src/components/hunts/hunt-list.jsx | 3 +- http_comms/comms.go | 3 +- http_comms/websocket_connection.go | 10 +-- server/comms.go | 27 ++------ server/server.go | 3 + server/websocket.go | 5 +- services/client_info/client_info.go | 25 +++---- services/client_info/tasks.go | 7 +- services/hunt_dispatcher/hunt_dispatcher.go | 4 +- services/notebook/calculate_test.go | 8 +++ services/notebook/storage_test.go | 20 ++++-- services/notifications/debug.go | 21 ++++++ services/notifications/notifications.go | 12 ++++ services/scheduler/scheduler.go | 19 +++++- 24 files changed, 238 insertions(+), 149 deletions(-) create mode 100644 services/notifications/debug.go diff --git a/api/api.go b/api/api.go index 5b4446bc811..942520800a2 100644 --- a/api/api.go +++ b/api/api.go @@ -18,6 +18,7 @@ along with this program. If not, see . package api import ( + "context" "crypto/tls" "crypto/x509" "fmt" @@ -32,7 +33,6 @@ import ( "github.com/Velocidex/ordereddict" errors "github.com/go-errors/errors" "github.com/prometheus/client_golang/prometheus/promhttp" - context "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -713,6 +713,10 @@ func (self *ApiServer) GetArtifacts( for _, name := range in.Names { artifact, pres := repository.Get(ctx, org_config_obj, name) + if !pres { + continue + } + artifact_clone := proto.Clone(artifact).(*artifacts_proto.Artifact) for _, s := range artifact_clone.Sources { s.Queries = nil @@ -1238,8 +1242,9 @@ func StartMonitoringService( <-ctx.Done() logger.Info("Shutting down Prometheus monitoring service") - timeout_ctx, cancel := context.WithTimeout( - context.Background(), 10*time.Second) + timeout_ctx, cancel := context.WithTimeoutCause( + context.Background(), 10*time.Second, + errors.New("Monitoring Service deadline reached")) defer cancel() err := server.Shutdown(timeout_ctx) diff --git a/api/builder.go b/api/builder.go index e5981538a1e..6842d9c6f7a 100644 --- a/api/builder.go +++ b/api/builder.go @@ -383,8 +383,9 @@ func StartFrontendHttps( server_obj.Info("Shutting down frontend") atomic.StoreInt32(&server_obj.Healthy, 0) - time_ctx, cancel := context.WithTimeout( - context.Background(), 10*time.Second) + time_ctx, cancel := context.WithTimeoutCause( + context.Background(), 10*time.Second, + errors.New("Deadline exceeded shuttin down frontend")) defer cancel() server.SetKeepAlivesEnabled(false) @@ -580,8 +581,9 @@ func StartFrontendWithAutocert( server_obj.Info("Stopping Frontend Server") atomic.StoreInt32(&server_obj.Healthy, 0) - timeout_ctx, cancel := context.WithTimeout( - context.Background(), 10*time.Second) + timeout_ctx, cancel := context.WithTimeoutCause( + context.Background(), 10*time.Second, + errors.New("Deadline exceeded shuttin down frontend")) defer cancel() server.SetKeepAlivesEnabled(false) @@ -641,8 +643,9 @@ func StartHTTPGUI( <-ctx.Done() logger.Info("Stopping GUI Server") - timeout_ctx, cancel := context.WithTimeout( - context.Background(), 10*time.Second) + timeout_ctx, cancel := context.WithTimeoutCause( + context.Background(), 10*time.Second, + errors.New("Deadline exceeded shuttin down GUI")) defer cancel() server.SetKeepAlivesEnabled(false) @@ -720,8 +723,9 @@ func StartSelfSignedGUI( <-ctx.Done() logger.Info("Stopping GUI Server") - timeout_ctx, cancel := context.WithTimeout( - context.Background(), 10*time.Second) + timeout_ctx, cancel := context.WithTimeoutCause( + context.Background(), 10*time.Second, + errors.New("Deadline exceeded shuttin down GUI")) defer cancel() server.SetKeepAlivesEnabled(false) diff --git a/api/query.go b/api/query.go index 0ec432fd845..4cca8660a32 100644 --- a/api/query.go +++ b/api/query.go @@ -1,23 +1,24 @@ /* - Velociraptor - Dig Deeper - Copyright (C) 2019-2024 Rapid7 Inc. +Velociraptor - Dig Deeper +Copyright (C) 2019-2024 Rapid7 Inc. - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package api import ( + "context" "fmt" "io" "log" @@ -30,7 +31,6 @@ import ( errors "github.com/go-errors/errors" "github.com/sirupsen/logrus" - context "golang.org/x/net/context" "www.velocidex.com/golang/velociraptor/actions" actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" api_proto "www.velocidex.com/golang/velociraptor/api/proto" @@ -111,8 +111,9 @@ func streamQuery( // Implement timeout if arg.Timeout > 0 { start := time.Now() - timed_ctx, timed_cancel := context.WithTimeout(subctx, - time.Second*time.Duration(arg.Timeout)) + timed_ctx, timed_cancel := context.WithTimeoutCause(subctx, + time.Second*time.Duration(arg.Timeout), + errors.New("Query API timeout reached")) wg.Add(1) go func() { diff --git a/bin/query.go b/bin/query.go index 7ce1f729ba0..cc0a7c96b96 100644 --- a/bin/query.go +++ b/bin/query.go @@ -27,6 +27,7 @@ import ( "time" "github.com/Velocidex/ordereddict" + errors "github.com/go-errors/errors" kingpin "gopkg.in/alecthomas/kingpin.v2" "www.velocidex.com/golang/velociraptor/actions" actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" @@ -390,8 +391,9 @@ func doQuery() error { if *query_command_collect_timeout > 0 { start := time.Now() - timed_ctx, timed_cancel := context.WithTimeout(ctx, - time.Second*time.Duration(*query_command_collect_timeout)) + timed_ctx, timed_cancel := context.WithTimeoutCause(ctx, + time.Second*time.Duration(*query_command_collect_timeout), + errors.New("Query: deadline reached")) go func() { select { diff --git a/executor/pool.go b/executor/pool.go index 00291072351..7860e439ca6 100644 --- a/executor/pool.go +++ b/executor/pool.go @@ -309,6 +309,14 @@ func (self *PoolClientExecutor) ProcessRequest( ctx context.Context, message *crypto_proto.VeloMessage) { + // Handle FlowStatsRequest specially - we just pretend this client + // does not support this feature. + if message.FlowStatsRequest != nil { + responder.MakeErrorResponse(self.Outbound, message.SessionId, + "Unsupported in Pool Client") + return + } + if message.UpdateEventTable != nil { self.delegate.maybeUpdateEventTable(ctx, message) return diff --git a/flows/client_flow_runner.go b/flows/client_flow_runner.go index c38f0971b25..a5bcda56828 100644 --- a/flows/client_flow_runner.go +++ b/flows/client_flow_runner.go @@ -258,7 +258,19 @@ func (self *ClientFlowRunner) removeInflightChecks( Set("ClearFlows", true), "Server.Internal.ClientScheduled") - return nil + // Update the client's in flight flow tracker on the local system + // as well. This helps to update this record ASAP before waiting + // for the minion message to arrive. + client_info_manager, err := services.GetClientInfoManager(self.config_obj) + if err != nil { + return err + } + + return client_info_manager.Modify(ctx, client_id, + func(client_info *services.ClientInfo) (*services.ClientInfo, error) { + client_info.InFlightFlows = nil + return client_info, nil + }) } func (self *ClientFlowRunner) ProcessSingleMessage( @@ -493,6 +505,12 @@ func (self *ClientFlowRunner) FlowStats( return err } + // Update the client's in flight flow tracker. + client_info_manager, err := services.GetClientInfoManager(self.config_obj) + if err != nil { + return err + } + // If this is the final response, then we will notify a flow // completion. if msg.FlowComplete { @@ -502,12 +520,20 @@ func (self *ClientFlowRunner) FlowStats( Set("Flow", stats). Set("FlowId", flow_id). Set("ClientId", client_id)) - } - // Update the client's in flight flow tracker. - client_info_manager, err := services.GetClientInfoManager(self.config_obj) - if err != nil { - return err + // Immediately remove this flow from the local InFlightFlows + // so we dont schedule it again. + return client_info_manager.Modify(ctx, client_id, + func(client_info *services.ClientInfo) (*services.ClientInfo, error) { + if client_info.InFlightFlows == nil { + client_info.InFlightFlows = make(map[string]int64) + } + + // Update the timestamp that we last received a stats + // update from this flow. + delete(client_info.InFlightFlows, flow_id) + return client_info, nil + }) } return client_info_manager.Modify(ctx, client_id, @@ -515,6 +541,9 @@ func (self *ClientFlowRunner) FlowStats( if client_info.InFlightFlows == nil { client_info.InFlightFlows = make(map[string]int64) } + + // Update the timestamp that we last received a stats + // update from this flow. client_info.InFlightFlows[flow_id] = utils.GetTime().Now().Unix() return client_info, nil }) diff --git a/go.mod b/go.mod index fd86fb00587..8675c973c4e 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/Velocidex/go-elasticsearch/v7 v7.3.1-0.20191001125819-fee0ef9cac6b github.com/Velocidex/go-magic v0.0.0-20211018155418-c5dc48282f28 github.com/Velocidex/go-yara v1.1.10-0.20240309155455-3f491847cec9 - github.com/Velocidex/grpc-go-pool v1.2.2-0.20211129003310-ece3b3fe13f4 + github.com/Velocidex/grpc-go-pool v1.2.2-0.20241016164850-ff0cb80037a8 github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a github.com/Velocidex/pkcs7 v0.0.0-20230220112103-d4ed02e1862a github.com/Velocidex/sflags v0.3.1-0.20231011011525-620ab7ca8617 @@ -37,7 +37,7 @@ require ( github.com/google/btree v1.1.2 github.com/google/rpmpack v0.5.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.6.0 github.com/gorilla/csrf v1.6.2 github.com/gorilla/schema v1.4.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 @@ -66,7 +66,6 @@ require ( github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 github.com/robertkrimen/otto v0.3.0 github.com/russross/blackfriday/v2 v2.1.0 - github.com/sebdah/goldie v1.0.0 github.com/sebdah/goldie/v2 v2.5.3 github.com/sergi/go-diff v1.2.0 github.com/sirupsen/logrus v1.8.1 @@ -74,16 +73,16 @@ require ( github.com/xor-gate/ar v0.0.0-20170530204233-5c72ae81e2b7 // indirect github.com/xor-gate/debpkg v1.0.0 go.starlark.net v0.0.0-20230925163745-10651d5192ab - golang.org/x/crypto v0.26.0 + golang.org/x/crypto v0.28.0 golang.org/x/mod v0.17.0 - golang.org/x/net v0.27.0 - golang.org/x/sys v0.25.0 + golang.org/x/net v0.30.0 + golang.org/x/sys v0.26.0 golang.org/x/text v0.19.0 golang.org/x/time v0.3.0 google.golang.org/api v0.146.0 google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c // indirect - google.golang.org/grpc v1.58.3 - google.golang.org/protobuf v1.34.2 + google.golang.org/grpc v1.67.1 + google.golang.org/protobuf v1.35.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df @@ -149,7 +148,7 @@ require ( github.com/vincent-petithory/dataurl v1.0.0 github.com/virtuald/go-paniclog v0.0.0-20190812204905-43a7fa316459 golang.org/x/oauth2 v0.22.0 - google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c + google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -159,7 +158,7 @@ require ( require ( cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/iam v1.1.2 // indirect github.com/360EntSecGroup-Skylar/excelize v1.4.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect @@ -187,7 +186,7 @@ require ( github.com/beevik/etree v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cavaliergopher/cpio v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/crewjam/httperr v0.2.0 // indirect github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect @@ -195,9 +194,8 @@ require ( github.com/geoffgarside/ber v1.1.0 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f // indirect - github.com/golang/glog v1.1.0 // indirect + github.com/golang/glog v1.2.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect @@ -239,9 +237,9 @@ require ( go.uber.org/goleak v1.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/term v0.23.0 // indirect + golang.org/x/term v0.25.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e // indirect ) diff --git a/go.sum b/go.sum index fbfbe12388e..fb0344f5072 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= -cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/kms v1.15.2 h1:lh6qra6oC4AyWe5fUUUBe/S27k12OHAleOOOw6KakdE= @@ -68,8 +68,6 @@ github.com/Velocidex/go-ext4 v0.0.0-20240608083317-8dd00855b069 h1:9OVXv4dSnn23i github.com/Velocidex/go-ext4 v0.0.0-20240608083317-8dd00855b069/go.mod h1:Sbqqh1t+nYXmNWw0dZC8LOIxP7z5Wg94SP+4Ej1QZqg= github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a h1:dWHPlB3C86vh+M5P14dZxF6Hh8o2/u8FTRF/bs2EM+Q= github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a/go.mod h1:g74FCv59tsVP48V2o1eyIK8aKbNKPLJIJ+HuiUPVc6E= -github.com/Velocidex/go-journalctl v0.0.0-20240721235158-d5e988dd16d2 h1:NKVH1COab/TNZEkovPt6D8cRVP7zT63AGQTwkFGpsZg= -github.com/Velocidex/go-journalctl v0.0.0-20240721235158-d5e988dd16d2/go.mod h1:5WxXsCtLdEvnc4FsFa+QfMwOWYkfey3nlQbPssZWqjc= github.com/Velocidex/go-journalctl v0.0.0-20241004063153-cc1c858415bd h1:CSTW6zYoG1IFxaGM3N42wSwruigV1xZ4gNzjLgb2xIc= github.com/Velocidex/go-journalctl v0.0.0-20241004063153-cc1c858415bd/go.mod h1:5WxXsCtLdEvnc4FsFa+QfMwOWYkfey3nlQbPssZWqjc= github.com/Velocidex/go-magic v0.0.0-20211018155418-c5dc48282f28 h1:3FMhXfGzZR4oNHmV8NizrviyaTv+2SmLuj+43cMJCUQ= @@ -78,16 +76,14 @@ github.com/Velocidex/go-mscfb v0.0.0-20240618091452-31f4ccc54002 h1:FWeeVb/x+Xva github.com/Velocidex/go-mscfb v0.0.0-20240618091452-31f4ccc54002/go.mod h1:YvYAfyK6Jg2WIaqvK42KPmVDfU8FSVxoSiZSVJfihDo= github.com/Velocidex/go-vhdx v0.0.0-20240601014259-b204818c95fd h1:znnjIQdOK6aqsG/crrEBAWBJzYdg1+jn/IGLdozC0qU= github.com/Velocidex/go-vhdx v0.0.0-20240601014259-b204818c95fd/go.mod h1:lBB+XfppHmYxIfnWDszC78x1iMq8088nMnMYQFQovM4= -github.com/Velocidex/go-vmdk v0.0.0-20240907200901-a6d9154b7c6b h1:V8+kHpemgk+lWbTe89mXGN9T5eS36uheUQpf1OJ+VFs= -github.com/Velocidex/go-vmdk v0.0.0-20240907200901-a6d9154b7c6b/go.mod h1:9avYIdKoZoSGjOGeJII/mqDVtqxDcdDdcK4tiPI82jE= github.com/Velocidex/go-vmdk v0.0.0-20240909080044-e373986b6517 h1:6/UGPskHUCHknG8RSWWx0Kzx3x7Y4n756LUMWLAB+m8= github.com/Velocidex/go-vmdk v0.0.0-20240909080044-e373986b6517/go.mod h1:GLNH1h3hHOeOWzFIDdBGXUtROaMZR1KF4U3DnoQdxHw= github.com/Velocidex/go-yara v1.1.10-0.20240309155455-3f491847cec9 h1:m+sfPvJnRTDL9EoJdlBOGa1dQlm0NPJgcXB/zpK/W7Y= github.com/Velocidex/go-yara v1.1.10-0.20240309155455-3f491847cec9/go.mod h1:5KKwF74avzF4GPZg37NoWfJiDgrBF5ASHH27g1p83zk= github.com/Velocidex/grok v0.0.1 h1:bOaqowsW9ecfJISQOM9K47Jk/ID9V945BLfKngUZEjo= github.com/Velocidex/grok v0.0.1/go.mod h1:tO25ta1WAuru0QA9ajcK0bAsk0gpWiT8so+fR4u3GZE= -github.com/Velocidex/grpc-go-pool v1.2.2-0.20211129003310-ece3b3fe13f4 h1:EfgJNtG9KBbbT8fhWIs0E+PP5prHixSE5/tQ1yzk7jI= -github.com/Velocidex/grpc-go-pool v1.2.2-0.20211129003310-ece3b3fe13f4/go.mod h1:rXXwht2ctScxF1tBYKG/9D0bT8mZXDlgTQPyypikbxc= +github.com/Velocidex/grpc-go-pool v1.2.2-0.20241016164850-ff0cb80037a8 h1:ezfYixd2bTfPGaLsSKl77aIje68WDgITm65bK0vF5Ak= +github.com/Velocidex/grpc-go-pool v1.2.2-0.20241016164850-ff0cb80037a8/go.mod h1:lyHshoS5LiGow9v99IvfFM8xmyOGoYKtjbcNQx20z4M= github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a h1:AeXPUzhU0yhID/v5JJEIkjaE85ASe+Vh4Kuv1RSLL+4= github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a/go.mod h1:ukJBuruT9b24pdgZwWDvOaCYHeS03B7oQPCUWh25bwM= github.com/Velocidex/ordereddict v0.0.0-20220107075049-3dbe58412844/go.mod h1:Y5Tfx5SKGOzkulpqfonrdILSPIuNg+GqKE/DhVJgnpg= @@ -201,8 +197,8 @@ github.com/bradleyjkemp/cupaloy/v2 v2.6.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1l github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM= github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -277,8 +273,8 @@ github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg= github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f/go.mod h1:ijRvpgDJDI262hYq/IQVYgf8hd8IHUs93Ol0kvMBAx4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -330,8 +326,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -646,8 +642,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -689,8 +685,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= @@ -743,18 +739,16 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -765,8 +759,6 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -807,18 +799,18 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c h1:ml3TAUoIIzQUtX88s/icpXCFW9lV5VwsuIuS1htNjKY= google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:MugzuwC+GYOxyF0XUGQvsT97bOgWCV7MM1XMc5FZv8E= -google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c h1:0RtEmmHjemvUXloH7+RuBSIw7n+GEHMOMY1CkGYnWq4= -google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:Wth13BrWMRN/G+guBLupKa6fslcWZv14R0ZKDRkNfY8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -832,8 +824,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= @@ -880,24 +872,18 @@ www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e h1:uf www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e/go.mod h1:jk+uZGukrJZWgnNH6q9tJLUnbugHEDPCQdIOmBBMXY4= www.velocidex.com/golang/evtx v0.2.1-0.20240730174545-3e4ff3d96433 h1:qrRlDit2WJgfGA4xjNq9/xdFJQGkrXfe1BuJRkZ41jA= www.velocidex.com/golang/evtx v0.2.1-0.20240730174545-3e4ff3d96433/go.mod h1:z0QWgpVDct1l+cHNq64vrSWdFuY6/BgrW2f/Qrc6oK4= -www.velocidex.com/golang/go-ese v0.2.1-0.20240207005444-85d57b555f8b h1:3pFfQuY3k0qViJDlLqmUfGP4YkQIl25Vc/Uq8Pl0qLA= -www.velocidex.com/golang/go-ese v0.2.1-0.20240207005444-85d57b555f8b/go.mod h1:6fC9T6UGLbM7icuA0ugomU5HbFC5XA5I30zlWtZT8YE= www.velocidex.com/golang/go-ese v0.2.1-0.20240919031214-2aa005106db2 h1:f7nj4NsyeMSrwiFd9XO/VfsZYt6o6FH1KJmmqlBZDgM= www.velocidex.com/golang/go-ese v0.2.1-0.20240919031214-2aa005106db2/go.mod h1:YKxCStqE15c6F/P81oCG0Y5oelDBah2hCdO6P+VPUIQ= www.velocidex.com/golang/go-ntfs v0.2.1-0.20240818145200-04736de821dc h1:eeL+RUEGr6/lYL8hJEbvugrF88I6W4pBaVtFa1falj4= www.velocidex.com/golang/go-ntfs v0.2.1-0.20240818145200-04736de821dc/go.mod h1:itvbHQcnLdTVIDY6fI3lR0zeBwXwBYBdUFtswE0x1vc= www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3 h1:W394TEIFuHFxHY8mzTJPHI5v+M+NLKEHmHn7KY/VpEM= www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3/go.mod h1:agYwYzeeytVtdwkRrvxZAjgIA8SCeM/Tg7Ym2/jBwmA= -www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a h1:iXYitl3w8pDTH8ltwra5Za9weXN5p8UAi87JoGf8SvU= -www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a/go.mod h1:UNIUmQhflpSTt7TH4o/6O/GiMCjSzIALXe9/zzTKFCw= www.velocidex.com/golang/go-prefetch v0.0.0-20240910051453-2385582c1c22 h1:Re+YlRCwkHESCIopk0WNLKXMnlnhALvoT4RiunT2qJE= www.velocidex.com/golang/go-prefetch v0.0.0-20240910051453-2385582c1c22/go.mod h1:UNIUmQhflpSTt7TH4o/6O/GiMCjSzIALXe9/zzTKFCw= www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe h1:o9jQWSwKTLhBeavfOk054/HK5yNi6Ni9VHQ6rxYZEi4= www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe/go.mod h1:R7IisRzDO7q5LVRJsCQf1xA50LrIavsPWzAjVE4THyY= www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09 h1:G1RWYBXP2lSzxKcrAU1YhiUlBetZ7hGIzIiWuuazvfo= www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU= -www.velocidex.com/golang/vfilter v0.0.0-20240812032614-1f95429e06af h1:klCJBImYvYAAfCy8STae7xju1o5pu2AUerBqPyymryM= -www.velocidex.com/golang/vfilter v0.0.0-20240812032614-1f95429e06af/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY= www.velocidex.com/golang/vfilter v0.0.0-20241009150353-76c3a28b1767 h1:XUBc9OV6JZuLjIuGSyRS5sZmkWWdfav8SazJBy3MNeI= www.velocidex.com/golang/vfilter v0.0.0-20241009150353-76c3a28b1767/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY= www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c h1:rL/It+Ig+mvIhmy9vl5gg5b6CX2J12x0v2SXIT2RoWE= diff --git a/gui/velociraptor/src/components/core/mode-yaml.jsx b/gui/velociraptor/src/components/core/mode-yaml.jsx index 9d13759cf34..c2d7e4066f4 100644 --- a/gui/velociraptor/src/components/core/mode-yaml.jsx +++ b/gui/velociraptor/src/components/core/mode-yaml.jsx @@ -6,16 +6,15 @@ export class YamlHighlightRules extends window.ace.acequire("ace/mode/yaml_highl this.$rules["start"] = [{ token : "keyword", - regex : /\s*-?\s*(name|type|description|choices|sources|parameters|author|reference|required_permissions|resources|tools|parameters|url|default|serve_locally|github_asset_regex|github_project|column_types|imports|export|notebook|template|timeout|ops_per_second|max_rows|max_upload_bytes|artifact_type|version|expected_hash):/, + regex : /\s*-?\s*(name|type|description|choices|sources|parameters|author|reference|required_permissions|resources|tools|parameters|url|default|serve_locally|github_asset_regex|github_project|column_types|imports|notebook|template|timeout|ops_per_second|max_rows|max_upload_bytes|artifact_type|version|expected_hash):/, }, { token: "keyword", - regex: /.*(precondition|query):\s*[|]?/, + regex: /.*(export|precondition|query):\s*[|]?/, onMatch: function(val, state, stack, line) { line = line.replace(/ #.*/, ""); var indent = /^\s*((:\s*)?-(\s*[^|>])?)?/.exec(line)[0] .replace(/\S\s*$/, "").length; var indentationIndicator = parseInt(/\d+[\s+-]*$/.exec(line)); - if (indentationIndicator) { indent += indentationIndicator - 1; this.next = "vql-start"; diff --git a/gui/velociraptor/src/components/core/table.jsx b/gui/velociraptor/src/components/core/table.jsx index 619debe3252..351a8e7b4dc 100644 --- a/gui/velociraptor/src/components/core/table.jsx +++ b/gui/velociraptor/src/components/core/table.jsx @@ -912,6 +912,10 @@ export function formatColumns(columns, env, column_formatter) { case undefined: break; + case "hidden": + x.type = null; + break; + default: console.log("Unsupported column type " + x.type); x.type = null; diff --git a/gui/velociraptor/src/components/hunts/hunt-list.jsx b/gui/velociraptor/src/components/hunts/hunt-list.jsx index db46c401d85..303da610966 100644 --- a/gui/velociraptor/src/components/hunts/hunt-list.jsx +++ b/gui/velociraptor/src/components/hunts/hunt-list.jsx @@ -77,7 +77,8 @@ class ModifyHuntDialog extends React.Component { let hunt = this.props.hunt; if(hunt) { - this.setState({tags: [...this.props.hunt.tags]}); + let tags = this.props.hunt.tags || []; + this.setState({tags: [...tags]}); } }; }); diff --git a/http_comms/comms.go b/http_comms/comms.go index 4b2e152b040..a0bea59d1c1 100644 --- a/http_comms/comms.go +++ b/http_comms/comms.go @@ -969,7 +969,8 @@ func (self *NotificationReader) GetMessageList() *crypto_proto.MessageList { // Attach the Server.Internal.ClientInfo message very // infrequently. now := utils.Now() - if now.Add(-self.last_update_period).After(self.last_update_time) { + if self.last_update_period > 0 && + now.Add(-self.last_update_period).After(self.last_update_time) { self.last_update_time = now client_info := self.executor.GetClientInfo() diff --git a/http_comms/websocket_connection.go b/http_comms/websocket_connection.go index f8da908880c..a7ce52b33a8 100644 --- a/http_comms/websocket_connection.go +++ b/http_comms/websocket_connection.go @@ -238,7 +238,7 @@ type WebSocketConnectionFactoryForTests struct { Connections map[string]*WebSocketConnection } -func (self WebSocketConnectionFactoryForTests) NewWebSocketConnection( +func (self *WebSocketConnectionFactoryForTests) NewWebSocketConnection( ctx context.Context, transport *HTTPClientWithWebSocketTransport, req *http.Request) (*WebSocketConnection, error) { @@ -251,7 +251,7 @@ func (self WebSocketConnectionFactoryForTests) NewWebSocketConnection( return conn, err } -func (self WebSocketConnectionFactoryForTests) Shutdown() { +func (self *WebSocketConnectionFactoryForTests) Shutdown() { self.mu.Lock() defer self.mu.Unlock() @@ -260,7 +260,7 @@ func (self WebSocketConnectionFactoryForTests) Shutdown() { } } -func (self WebSocketConnectionFactoryForTests) GetConn(key string) (*WebSocketConnection, bool) { +func (self *WebSocketConnectionFactoryForTests) GetConn(key string) (*WebSocketConnection, bool) { self.mu.Lock() defer self.mu.Unlock() @@ -268,8 +268,8 @@ func (self WebSocketConnectionFactoryForTests) GetConn(key string) (*WebSocketCo return con, pres } -func NewWebSocketConnectionFactoryForTests() WebSocketConnectionFactoryForTests { - return WebSocketConnectionFactoryForTests{ +func NewWebSocketConnectionFactoryForTests() *WebSocketConnectionFactoryForTests { + return &WebSocketConnectionFactoryForTests{ Connections: make(map[string]*WebSocketConnection), } } diff --git a/server/comms.go b/server/comms.go index 8af4ff858f0..e45c9ca76ce 100644 --- a/server/comms.go +++ b/server/comms.go @@ -194,27 +194,6 @@ func server_pem(config_obj *config_proto.Config, server_obj *Server) http.Handle }) } -// Redirect client to another active frontend. -/* Experimental code disabled for now. -func maybeRedirectFrontend(handler string, w http.ResponseWriter, r *http.Request) bool { - _, pres := r.URL.Query()["r"] - if pres { - return false - } - - redirect_url, ok := services.Frontend.GetFrontendURL() - if ok { - redirectedFrontendCounter.Inc() - // We should redirect to another frontend. - http.Redirect(w, r, redirect_url, 301) - return true - } - - // Handle request ourselves. - return false -} -*/ - // Read the message from the client carefully. Due to concurrency // control we want to dismiss slow clients as soon as possible since // processing them is taking a concurrency slot and causes slow down @@ -303,7 +282,9 @@ func receive_client_messages( // Allow a limited time to read from the client because this // is the hot path. - ctx, cancel := context.WithTimeout(req.Context(), 600*time.Second) + ctx, cancel := context.WithTimeoutCause( + req.Context(), 600*time.Second, + errors.New("receive_client_messages: deadline reached processing message")) defer cancel() // If the client connection drops we close the reader. @@ -491,7 +472,7 @@ func send_client_messages( return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { ctx := req.Context() - // Keep track of currently connected clients - this account + // Keep track of currently connected clients - this accounts // for clients using http and websocket. currentConnections.Inc() defer currentConnections.Dec() diff --git a/server/server.go b/server/server.go index 2e5910caa5c..3d1059c89f6 100644 --- a/server/server.go +++ b/server/server.go @@ -274,6 +274,9 @@ func (self *Server) Process( } message_list := &crypto_proto.MessageList{} + + // Check if any messages are queued for the client. This also + // checks for any outstanding status checks. if drain_requests_for_client { tasks, err := client_info_manager.GetClientTasks(ctx, message_info.Source) if err == nil { diff --git a/server/websocket.go b/server/websocket.go index c670b701e8e..6f4de0f0414 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -161,8 +161,9 @@ func ws_receive_client_messages( // client disconnects quickly the request context will be // cancelled and aborted, but we do not want this to // interrupt actually processing the message. - subctx, cancel := context.WithTimeout(context.Background(), - 60*time.Second) + subctx, cancel := context.WithTimeoutCause( + context.Background(), 60*time.Second, + errors.New("Websocket: deadline reached processing message")) _, _, err = server_obj.Process(subctx, message_info, DoNotDrainRequestsForClient) diff --git a/services/client_info/client_info.go b/services/client_info/client_info.go index ddc5b2453a8..962007e47db 100644 --- a/services/client_info/client_info.go +++ b/services/client_info/client_info.go @@ -282,28 +282,21 @@ func (self *ClientInfoManager) ProcessFlowCompletion( return nil } - err := self.storage.Modify(ctx, config_obj, client_id, + // The flow is completed, remove it from the flow completion. + return self.storage.Modify(ctx, config_obj, client_id, func(client_info *services.ClientInfo) (*services.ClientInfo, error) { - if client_info == nil || client_info.InFlightFlows == nil { + if client_info == nil { return nil, utils.NotFoundError } - delete(client_info.InFlightFlows, flow_id) + if client_info.InFlightFlows != nil { + delete(client_info.InFlightFlows, flow_id) + } return client_info, nil }) - if err != nil { - return err - } - - notifier, err := services.GetNotifier(self.config_obj) - if err != nil { - return err - } - - return notifier.NotifyListener( - ctx, self.config_obj, client_id, "ClientInfoManager") } +// Messages from other nodes to update the client info record. func (self *ClientInfoManager) ProcessInFlightNotifications( ctx context.Context, config_obj *config_proto.Config, row *ordereddict.Dict) error { @@ -595,7 +588,9 @@ func NewClientInfoManager( utils.DlvBreak() // When we shut down make sure to save the snapshot. - subctx, cancel := context.WithTimeout(context.Background(), 100*time.Second) + subctx, cancel := context.WithTimeoutCause( + context.Background(), 100*time.Second, + errors.New("ClientInfoService: deadline reached saving snapshot")) defer cancel() service.storage.SaveSnapshot(subctx, config_obj, SYNC_UPDATE) diff --git a/services/client_info/tasks.go b/services/client_info/tasks.go index 41c8725493d..1c516f09394 100644 --- a/services/client_info/tasks.go +++ b/services/client_info/tasks.go @@ -412,11 +412,16 @@ func (self *ClientInfoManager) GetClientTasks( // Check up on in flight flows every 60 sec at least // (could be more depending on poll). for k, v := range client_info.InFlightFlows { - if now-v > 10 { + if now-v > 60 { inflight_notifications = append(inflight_notifications, k) } } + // Update the time to ensure we dont send these too often. + for _, k := range inflight_notifications { + client_info.InFlightFlows[k] = utils.GetTime().Now().Unix() + } + // Reset the HasTasks flag client_info.HasTasks = false return client_info, nil diff --git a/services/hunt_dispatcher/hunt_dispatcher.go b/services/hunt_dispatcher/hunt_dispatcher.go index e303ac5d589..1aa5fec5ad9 100644 --- a/services/hunt_dispatcher/hunt_dispatcher.go +++ b/services/hunt_dispatcher/hunt_dispatcher.go @@ -513,7 +513,9 @@ func NewHuntDispatcher( select { case <-ctx.Done(): // Give at most 10 seconds for shutdown. - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeoutCause( + context.Background(), 10*time.Second, + errors.New("HuntDispatcher: deadline reached shutting down")) defer cancel() service.Close(ctx) return diff --git a/services/notebook/calculate_test.go b/services/notebook/calculate_test.go index 8707de40c82..c29af83eb63 100644 --- a/services/notebook/calculate_test.go +++ b/services/notebook/calculate_test.go @@ -12,6 +12,7 @@ import ( "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/services" "www.velocidex.com/golang/velociraptor/services/notebook" + "www.velocidex.com/golang/velociraptor/services/scheduler" "www.velocidex.com/golang/velociraptor/utils" "www.velocidex.com/golang/velociraptor/vtesting" "www.velocidex.com/golang/velociraptor/vtesting/assert" @@ -74,6 +75,13 @@ func (self *NotebookManagerTestSuite) _TestNotebookManagerUpdateCell(r *assert.R closer := utils.SetIdGenerator(&gen) defer closer() + scheduler_service, err := services.GetSchedulerService(self.ConfigObj) + assert.NoError(self.T(), err) + + vtesting.WaitUntil(2*time.Second, r, func() bool { + return scheduler_service.(*scheduler.Scheduler).AvailableWorkers() > 0 + }) + notebook_manager, err := services.GetNotebookManager(self.ConfigObj) assert.NoError(r, err) diff --git a/services/notebook/storage_test.go b/services/notebook/storage_test.go index be093d250a5..ae9a1117cb8 100644 --- a/services/notebook/storage_test.go +++ b/services/notebook/storage_test.go @@ -6,6 +6,7 @@ import ( api_proto "www.velocidex.com/golang/velociraptor/api/proto" "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/services" + "www.velocidex.com/golang/velociraptor/services/scheduler" "www.velocidex.com/golang/velociraptor/vtesting" "www.velocidex.com/golang/velociraptor/vtesting/assert" ) @@ -19,17 +20,22 @@ func (self *NotebookManagerTestSuite) TestNotebookStorage() { assert.Equal(self.T(), 0, len(notebooks)) - var global_notebook *api_proto.NotebookMetadata + scheduler_service, err := services.GetSchedulerService(self.ConfigObj) + assert.NoError(self.T(), err) vtesting.WaitUntil(2*time.Second, self.T(), func() bool { - // Create a notebook the usual way. - global_notebook, err = notebook_manager.NewNotebook( - self.Ctx, "admin", &api_proto.NotebookMetadata{ - Name: "Test Global Notebook", - }) - return err == nil + return scheduler_service.(*scheduler.Scheduler).AvailableWorkers() > 0 }) + var global_notebook *api_proto.NotebookMetadata + + // Create a notebook the usual way. + global_notebook, err = notebook_manager.NewNotebook( + self.Ctx, "admin", &api_proto.NotebookMetadata{ + Name: "Test Global Notebook", + }) + assert.NoError(self.T(), err) + // Now create a flow notebook - these have pre-determined ID _, err = notebook_manager.NewNotebook( self.Ctx, "admin", &api_proto.NotebookMetadata{ diff --git a/services/notifications/debug.go b/services/notifications/debug.go new file mode 100644 index 00000000000..01a39750795 --- /dev/null +++ b/services/notifications/debug.go @@ -0,0 +1,21 @@ +package notifications + +import ( + "context" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/services" + "www.velocidex.com/golang/velociraptor/utils" + "www.velocidex.com/golang/vfilter" +) + +func (self *Notifier) WriteProfile(ctx context.Context, + scope vfilter.Scope, output_chan chan vfilter.Row) { + + for _, client_id := range self.ListClients() { + output_chan <- ordereddict.NewDict(). + Set("OrgId", utils.GetOrgId(self.config_obj)). + Set("ClientId", client_id). + Set("Hostname", services.GetHostname(ctx, self.config_obj, client_id)) + } +} diff --git a/services/notifications/notifications.go b/services/notifications/notifications.go index 5dac16bb9a5..956aa884ecd 100644 --- a/services/notifications/notifications.go +++ b/services/notifications/notifications.go @@ -27,6 +27,7 @@ import ( "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/notifications" "www.velocidex.com/golang/velociraptor/services" + "www.velocidex.com/golang/velociraptor/services/debug" "www.velocidex.com/golang/velociraptor/services/journal" "www.velocidex.com/golang/velociraptor/utils" ) @@ -71,6 +72,8 @@ type Notifier struct { uuid int64 client_connection_tracker map[string]*tracker + + config_obj *config_proto.Config } // The notifier service watches for events from @@ -87,6 +90,7 @@ func NewNotificationService( notification_pool: notifications.NewNotificationPool(ctx, wg), uuid: utils.GetGUID(), client_connection_tracker: make(map[string]*tracker), + config_obj: config_obj, } logger := logging.GetLogger(config_obj, &logging.FrontendComponent) @@ -149,6 +153,14 @@ func NewNotificationService( } }() + debug.RegisterProfileWriter(debug.ProfileWriterInfo{ + Name: "notifier-" + utils.GetOrgId(config_obj), + Description: fmt.Sprintf( + "Information about directly connected clients for org %v.", + services.GetOrgName(config_obj)), + ProfileWriter: self.WriteProfile, + }) + return self, nil } diff --git a/services/scheduler/scheduler.go b/services/scheduler/scheduler.go index 600e786fea2..85259a138f2 100644 --- a/services/scheduler/scheduler.go +++ b/services/scheduler/scheduler.go @@ -133,6 +133,22 @@ func (self *Scheduler) WriteProfile(ctx context.Context, self.mu.Unlock() } +func (self *Scheduler) AvailableWorkers() int { + count := 0 + + self.mu.Lock() + + for _, workers := range self.queues { + for _, w := range workers { + if !w.IsBusy() { + count++ + } + } + } + self.mu.Unlock() + return count +} + func (self *Scheduler) Schedule(ctx context.Context, job services.SchedulerJob) (chan services.JobResponse, error) { @@ -176,7 +192,8 @@ func (self *Scheduler) Schedule(ctx context.Context, // Give up after 10 seconds. if utils.GetTime().Now().Sub(start) > wait_time { - return nil, fmt.Errorf("No workers available on queue %v!", job.Queue) + return nil, fmt.Errorf( + "No workers available on queue %v!", job.Queue) } // Try again soon