Fixed test

artifacts/definitions/Windows/ETW/DNS.yaml

@@ -2,8 +2,8 @@ name: Windows.ETW.DNS
 author: Matt Green - @mgreen27
 description: |
   This artifact monitors DNS queries using ETW.
-  
-  There are several filteres availible to the user to filter out and target with 
+
+  There are several filteres availible to the user to filter out and target with
   regex, by default duplicate DNSCache requests are filtered out.
 
 type: CLIENT_EVENT
@@ -29,32 +29,34 @@ parameters:
     description: "Commandline to filter out. Typically we do not want Dnscache events."
     default: 'svchost.exe -k NetworkService -p -s Dnscache$'
     type: regex
-    
-    
+
+
 sources:
   - precondition:
       SELECT OS From info() where OS = 'windows'
-      
+
     query: |
       LET RecentProcesses = SELECT * FROM fifo(query={
-                SELECT System.TimeStamp AS CreateTime, 
+                SELECT System.TimeStamp AS CreateTime,
                     EventData.ImageName AS ImageName,
                     int(int=EventData.ProcessID) AS Pid,
                     EventData.MandatoryLabel AS MandatoryLabel,
                     EventData.ProcessTokenElevationType AS ProcessTokenElevationType,
                     EventData.ProcessTokenIsElevated AS TokenIsElevated
-                FROM watch_etw(guid="{22fb2cd6-0e7b-422b-a0c7-2fad1fd0e716}", any=0x10)
-                WHERE System.ID = 1   
+                FROM watch_etw(
+                  description="Microsoft-Windows-Kernel-Process/Analytic",
+                  guid="{22fb2cd6-0e7b-422b-a0c7-2fad1fd0e716}", any=0x10)
+                WHERE System.ID = 1
             }, max_rows=1000, max_age=60)
-        
+
       -- Query it once to materialize the FIFO
       LET _ <= SELECT * FROM RecentProcesses
-        
+
       LET GetProcessInfo(TargetPid) = SELECT *, ThreadId as ProcessThreadId
         FROM switch(
             -- First try to get the pid directly
             a={
-                SELECT 
+                SELECT
                     Name, Pid, CreateTime,
                     Exe as ImageName,
                     CommandLine,
@@ -71,14 +73,14 @@ sources:
                     ImageName,
                     Null as CommandLine,
                     Null as Username,
-                    if(condition= TokenIsElevated="0", 
-                        then= false, 
+                    if(condition= TokenIsElevated="0",
+                        then= false,
                         else= true ) as TokenIsElevated
                 FROM RecentProcesses
                 WHERE Pid = TargetPid
                 LIMIT 1
             })
-        
+
         SELECT System.TimeStamp AS EventTime,
             EventData.QueryName AS Query,
             get(item=dict(
@@ -132,10 +134,12 @@ sources:
                     member=str(str=EventData.QueryType)) AS Type,
                EventData.QueryResults AS Answer,
                GetProcessInfo(TargetPid=System.ProcessID,ThreadId=System.ThreadID)[0] as Process
-        FROM watch_etw(guid="{1C95126E-7EEA-49A9-A3FE-A378B03DDB4D}")
+        FROM watch_etw(
+          description="Microsoft-Windows-DNS-Client",
+          guid="{1C95126E-7EEA-49A9-A3FE-A378B03DDB4D}")
         WHERE System.ID = 3008
-            AND Query AND Process AND Answer 
+            AND Query AND Process AND Answer
             AND NOT Process.CommandLine =~ CommandLineExclusion
             AND Process.ImageName =~ ImageRegex
             AND Query =~ QueryRegex
-            AND Answer =~ AnswerRegex
+            AND Answer =~ AnswerRegex

artifacts/definitions/Windows/ETW/DNSQueriesServer.yaml

@@ -31,7 +31,9 @@ sources:
                EventData.Source as Source,
                EventData.TCP as TCP,
                EventData.XID as XID
-        FROM watch_etw(guid="{EB79061A-A566-4698-9119-3ED2807060E7}")
+        FROM watch_etw(
+          description="EventLog-Microsoft-Windows-DNSServer-Analytical",
+          guid="{EB79061A-A566-4698-9119-3ED2807060E7}")
         WHERE EventData AND
               QNAME =~ QueryNameRegex AND
               Source =~ SourceIPRegex

artifacts/definitions/Windows/ETW/ETWSessions.yaml

@@ -27,5 +27,7 @@ sources:
         GUIDLookup(GUID=EventData.ProviderName)[0].Provider AS Provider,
         EventData.SessionName AS SessionName,
         System AS _System, EventData AS _EventData
-      FROM watch_etw(guid="{B675EC37-BDB6-4648-BC92-F3FDC74D3CA2}", all=0x400)
+      FROM watch_etw(
+         description='Microsoft-Windows-Kernel-EventTracing',
+         guid="{B675EC37-BDB6-4648-BC92-F3FDC74D3CA2}", all=0x400)
       WHERE System.ID IN (14, 15)

artifacts/definitions/Windows/ETW/EdgeURLs.yaml

@@ -28,5 +28,7 @@ sources:
              System.TimeStamp AS Timestamp,
              get(item=m, field=System.ProcessID) AS ProcInfo,
              get(member="EventData.URL") AS URL
-      FROM watch_etw(guid="{245F975D-909D-49ED-B8F9-9A75691D6B6B}")
+      FROM watch_etw(
+        description="Microsoft-Windows-URLMon",
+        guid="{245F975D-909D-49ED-B8F9-9A75691D6B6B}")
       WHERE ID = 805 AND URL =~ URLFilter

artifacts/definitions/Windows/ETW/FileCreation.yaml

@@ -29,7 +29,9 @@ sources:
 
     query: |
         LET Y = SELECT *
-          FROM watch_etw(guid="{EDD08927-9CC4-4E65-B970-C2560FB5C289}", any=4096)
+          FROM watch_etw(
+            description="Microsoft-Windows-Kernel-File/Analytic",
+            guid="{EDD08927-9CC4-4E65-B970-C2560FB5C289}", any=4096)
           WHERE EventData.FileName =~ FilePathFilter
 
         -- Implement a delay to ensure we get the process event stream

artifacts/definitions/Windows/ETW/Registry.yaml

@@ -47,7 +47,9 @@ sources:
     LET registry_access = SELECT System, EventData,
        get(item=EventLookup, field=str(str=System.ID)) AS EventType,
        get(item=Cache, field=EventData.KeyObject) || EventData.KeyName AS KeyName
-    FROM watch_etw(guid="{70EB4F03-C1DE-4F73-A051-33D13D5413BD}", any=0x7720)
+    FROM watch_etw(
+      description="Microsoft-Windows-Kernel-Registry",
+      guid="{70EB4F03-C1DE-4F73-A051-33D13D5413BD}", any=0x7720)
     WHERE System.ProcessID != getpid() -- exclude ourselves
         AND EventType  -- we only care about these events
         AND if(condition=System.ID in (1, 2, 4),

artifacts/definitions/Windows/ETW/WMIProcessCreate.yaml

@@ -20,7 +20,9 @@ sources:
          System.ID AS ID,
          System.TimeStamp AS Timestamp,
          get(member="EventData") AS EventData
-      FROM watch_etw(guid="{1418EF04-B0B4-4623-BF7E-D74AB47BBDAA}")
+      FROM watch_etw(
+        description="Microsoft-Windows-WMI-Activity",
+        guid="{1418EF04-B0B4-4623-BF7E-D74AB47BBDAA}")
       WHERE ID = 23
 
       SELECT ID, Timestamp, EventData.ClientMachine AS Hostname,

artifacts/definitions/Windows/Events/TrackProcesses.yaml

@@ -52,7 +52,9 @@ sources:
             SELECT * FROM foreach(row={
               SELECT *,
                      get(member='EventData') AS EventData
-              FROM watch_etw(guid='{5770385f-c22a-43e0-bf4c-06f5698ffbd9}')
+              FROM watch_etw(
+                guid='{5770385f-c22a-43e0-bf4c-06f5698ffbd9}',
+                description='Microsoft-Windows-Sysmon/Operational')
             }, query={
               SELECT * FROM switch(
               start={

artifacts/definitions/Windows/Sysinternals/SysmonLogForward.yaml

@@ -31,4 +31,6 @@ sources:
     SELECT System.ID AS ID,
            System.TimeStamp AS Timestamp,
            get(member='EventData') AS EventData
-    FROM watch_etw(guid='{5770385f-c22a-43e0-bf4c-06f5698ffbd9}')
+    FROM watch_etw(
+       description='Microsoft-Windows-Sysmon/Operational',
+       guid='{5770385f-c22a-43e0-bf4c-06f5698ffbd9}')

artifacts/testdata/server/testcases/profile.out.yaml

@@ -5,7 +5,6 @@ SELECT * FROM profile(metrics=TRUE) WHERE Line.name =~ "client_comms_current_con
    "name": "client_comms_current_connections",
    "help": "Number of currently connected clients.",
    "value": 0
-  },
-  "OSPath": ""
+  }
  }
 ]

services/users/list.go

@@ -2,6 +2,7 @@ package users
 
 import (
 	"context"
+	"sort"
 
 	"www.velocidex.com/golang/velociraptor/acls"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
@@ -120,6 +121,11 @@ func (self *UserManager) ListUsers(
 			continue
 		}
 
+		// Sort orgs for stable output
+		sort.Slice(user_record.Orgs, func(i, j int) bool {
+			return user_record.Orgs[i].Id < user_record.Orgs[j].Id
+		})
+
 		result = append(result, user_record)
 	}
 

vql/windows/etw/context.go

@@ -6,6 +6,7 @@ package etw
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/Velocidex/etw"
 	"github.com/Velocidex/ordereddict"
@@ -14,6 +15,13 @@ import (
 	"www.velocidex.com/golang/vfilter"
 )
 
+type Registration struct {
+	handles     []*Handle
+	count       int
+	description string
+	started     time.Time
+}
+
 // A Session context manages an ETW session:
 
 // 1. It can accomodate multiple registrations on multiple GUIDs.
@@ -30,7 +38,7 @@ type SessionContext struct {
 	wg sync.WaitGroup
 
 	// Registrations keyed by GUID
-	registrations map[string][]*Handle
+	registrations map[string]*Registration
 
 	resolve_map_info bool
 }
@@ -141,13 +149,14 @@ func (self *SessionContext) processEvent(e *etw.Event) {
 	event_guid := e.Header.ProviderID.String()
 
 	self.mu.Lock()
-	registrations, pres := self.registrations[event_guid]
+	registration, pres := self.registrations[event_guid]
 	if !pres {
 		self.mu.Unlock()
 		return
 	}
 
-	handlers := append([]*Handle{}, registrations...)
+	registration.count++
+	handlers := append([]*Handle{}, registration.handles...)
 	self.mu.Unlock()
 
 	event := ordereddict.NewDict().
@@ -172,10 +181,13 @@ func (self *SessionContext) Stats() []ProviderStat {
 	defer self.mu.Unlock()
 
 	result := []ProviderStat{}
-	for guid, registrations := range self.registrations {
+	for guid, registration := range self.registrations {
 		result = append(result, ProviderStat{
-			GUID:     guid,
-			Watchers: len(registrations),
+			GUID:        guid,
+			EventCount:  registration.count,
+			Description: registration.description,
+			Watchers:    len(registration.handles),
+			Started:     registration.started,
 		})
 	}
 
@@ -198,8 +210,13 @@ func (self *SessionContext) Register(
 		self.resolve_map_info = options.EnableMapInfo
 	}
 
-	registrations, pres := self.registrations[key]
+	registration, pres := self.registrations[key]
 	if !pres {
+		registration = &Registration{
+			description: options.Description,
+			started:     utils.GetTime().Now(),
+		}
+
 		// No one is currently watching this GUID, lets begin watching
 		// it.
 		session, err := self._Session(scope)
@@ -221,8 +238,8 @@ func (self *SessionContext) Register(
 		scope.Log("etw: Added provider %v to session %v", guid.String(), self.name)
 	}
 
-	registrations = append(registrations, handle)
-	self.registrations[key] = registrations
+	registration.handles = append(registration.handles, handle)
+	self.registrations[key] = registration
 
 	return func() {
 		self.DeregisterHandle(key, handle.id, guid, scope)
@@ -235,15 +252,15 @@ func (self *SessionContext) DeregisterHandle(
 	self.mu.Lock()
 	defer self.mu.Unlock()
 
-	registrations, pres := self.registrations[key]
+	registration, pres := self.registrations[key]
 	if !pres {
 		// No registrations for this provider
 		return
 	}
 
 	// Remove the handle from the registrations
-	new_reg := make([]*Handle, 0, len(registrations))
-	for _, handle := range registrations {
+	new_reg := make([]*Handle, 0, len(registration.handles))
+	for _, handle := range registration.handles {
 		if handle.id == id {
 			scope.Log("etw: Deregistering %v from session %v",
 				handle.guid.String(), self.name)
@@ -256,7 +273,7 @@ func (self *SessionContext) DeregisterHandle(
 	}
 
 	if len(new_reg) > 0 {
-		self.registrations[key] = new_reg
+		registration.handles = new_reg
 		return
 	}
 

vql/windows/etw/options.go

@@ -5,4 +5,7 @@ type ETWOptions struct {
 	Level                  int64
 	CaptureState           bool
 	EnableMapInfo          bool
+
+	// A description string to be associated with the registration.
+	Description string
 }

vql/windows/etw/stats.go

@@ -3,8 +3,13 @@
 
 package etw
 
+import "time"
+
 type ProviderStat struct {
 	SessionName string
 	GUID        string
+	Description string
+	EventCount  int
 	Watchers    int
+	Started     time.Time
 }

vql/windows/etw/watch_etw.go

@@ -25,6 +25,7 @@ type WatchETWArgs struct {
 	Timeout       uint64          `vfilter:"optional,field=timeout,doc=If provided we stop after this much time"`
 	CaptureState  bool            `vfilter:"optional,field=capture_state,doc=If true, capture the state of the provider when the event is triggered"`
 	EnableMapInfo bool            `vfilter:"optional,field=enable_map_info,doc=Resolving MapInfo with TdhGetEventMapInformation is very expensive and causes events to be dropped so we disabled it by default. Enable with this flag."`
+	Description   string          `vfilter:"optional,field=description,doc=Description for this GUID provider"`
 }
 
 type WatchETWPlugin struct{}
@@ -76,6 +77,7 @@ func (self WatchETWPlugin) Call(
 			Level:         arg.Level,
 			CaptureState:  arg.CaptureState,
 			EnableMapInfo: arg.EnableMapInfo,
+			Description:   arg.Description,
 		}
 
 		for {

vql/windows/etw/watcher.go

@@ -62,7 +62,7 @@ func (self *EventTraceWatcherService) SessionContext(
 	if !pres {
 		sessionContext = &SessionContext{
 			name:          name,
-			registrations: make(map[string][]*Handle),
+			registrations: make(map[string]*Registration),
 		}
 		self.sessions[name] = sessionContext
 	}
@@ -91,7 +91,10 @@ func writeMetrics(
 		output_chan <- ordereddict.NewDict().
 			Set("SessionName", s.SessionName).
 			Set("GUID", s.GUID).
-			Set("Watchers", s.Watchers)
+			Set("Description", s.Description).
+			Set("Watchers", s.Watchers).
+			Set("EventCount", s.EventCount).
+			Set("Started", s.Started)
 	}
 }