Support export logging

cmd/api/src/api/middleware/middleware.go

@@ -35,6 +35,7 @@ import (
 	"github.com/specterops/bloodhound/src/config"
 	"github.com/specterops/bloodhound/src/ctx"
 	"github.com/specterops/bloodhound/src/database"
+	"github.com/specterops/bloodhound/src/model"
 	"github.com/specterops/bloodhound/src/utils"
 	"github.com/unrolled/secure"
 )
@@ -140,7 +141,8 @@ func ContextMiddleware(next http.Handler) http.Handler {
 					Scheme: getScheme(request),
 					Host:   request.Host,
 				},
-				RequestIP: parseUserIP(request),
+				RequestedURL: model.AuditableURL(request.URL.String()),
+				RequestIP:    parseUserIP(request),
 			})
 
 			// Route the request with the embedded context

cmd/api/src/api/tools/trace.go

@@ -1,17 +1,17 @@
 // Copyright 2023 Specter Ops, Inc.
-// 
+//
 // Licensed under the Apache License, Version 2.0
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
-// 
+//
 //     http://www.apache.org/licenses/LICENSE-2.0
-// 
+//
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-// 
+//
 // SPDX-License-Identifier: Apache-2.0
 
 package tools
@@ -23,8 +23,8 @@ import (
 	"net/url"
 	"runtime/pprof"
 
-	"github.com/specterops/bloodhound/src/api"
 	"github.com/specterops/bloodhound/errors"
+	"github.com/specterops/bloodhound/src/api"
 )
 
 const (

cmd/api/src/ctx/ctx.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/specterops/bloodhound/src/auth"
+	"github.com/specterops/bloodhound/src/model"
 )
 
 // Use our own type rather than a primitive to avoid collisions
@@ -39,12 +40,13 @@ type RequestedWaitDuration struct {
 
 // Context holds contextual data that is passed around to functions. This is an extension to Golang's built in context.
 type Context struct {
-	StartTime time.Time
-	Timeout   RequestedWaitDuration
-	RequestID string
-	AuthCtx   auth.Context
-	Host      *url.URL
-	RequestIP string
+	StartTime    time.Time
+	Timeout      RequestedWaitDuration
+	RequestID    string
+	AuthCtx      auth.Context
+	Host         *url.URL
+	RequestedURL model.AuditableURL
+	RequestIP    string
 }
 
 func (s *Context) ConstructGoContext() context.Context {

cmd/api/src/daemons/api/toolapi/api.go

@@ -18,11 +18,13 @@ package toolapi
 
 import (
 	"context"
+	"net/http"
+	"net/http/pprof"
+	"time"
+
 	"github.com/specterops/bloodhound/dawgs/graph"
 	"github.com/specterops/bloodhound/src/bootstrap"
 	"github.com/specterops/bloodhound/src/database"
-	"net/http"
-	"time"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -47,7 +49,19 @@ func NewDaemon[DBType database.Database](ctx context.Context, connections bootst
 
 	router.Mount("/metrics", promhttp.Handler())
 
+	// Support normal pprof endpoints for easier consumption with standard tools
+	router.Mount("/debug/pprof/allocs", pprof.Handler("allocs"))
+	router.Mount("/debug/pprof/block", pprof.Handler("block"))
+	router.Mount("/debug/pprof/goroutine", pprof.Handler("goroutine"))
+	router.Mount("/debug/pprof/heap", pprof.Handler("heap"))
+	router.Mount("/debug/pprof/mutex", pprof.Handler("mutex"))
+	router.Mount("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+	router.Get("/debug/pprof/profile", pprof.Profile)
+	router.Get("/debug/pprof/trace", pprof.Trace)
+
+	// TODO: remove old trace handler when we can wire up acumen to handle the above pprof endpoints instead
 	router.Get("/trace", tools.NewTraceHandler())
+
 	router.Put("/graph-db/switch/pg", pgMigrator.SwitchPostgreSQL)
 	router.Put("/graph-db/switch/neo4j", pgMigrator.SwitchNeo4j)
 	router.Put("/pg-migration/start", pgMigrator.MigrationStart)

cmd/api/src/database/audit.go

@@ -111,6 +111,14 @@ func (s *BloodhoundDB) ListAuditLogs(before, after time.Time, offset, limit int,
 	return auditLogs, int(count), CheckError(result)
 }
 
+func (s *BloodhoundDB) MaybeAuditableTransaction(ctx context.Context, auditDisabled bool, auditEntry model.AuditEntry, f func(tx *gorm.DB) error, opts ...*sql.TxOptions) error {
+	if auditDisabled {
+		return s.db.Transaction(f, opts...)
+	} else {
+		return s.AuditableTransaction(ctx, auditEntry, f, opts...)
+	}
+}
+
 func (s *BloodhoundDB) AuditableTransaction(ctx context.Context, auditEntry model.AuditEntry, f func(tx *gorm.DB) error, opts ...*sql.TxOptions) error {
 	var (
 		commitID, err = uuid.NewV4()

cmd/api/src/model/audit.go

@@ -155,3 +155,11 @@ type AuditEntry struct {
 	Status   AuditEntryStatus
 	ErrorMsg string
 }
+
+type AuditableURL string
+
+func (s AuditableURL) AuditData() AuditData {
+	return AuditData{
+		"request_url": s,
+	}
+}

packages/go/analysis/ad/membership.go

@@ -38,8 +38,8 @@ func ResolveAllGroupMemberships(ctx context.Context, db graph.Database, addition
 		adGroupIDs []graph.ID
 
 		searchCriteria = []graph.Criteria{query.KindIn(query.Relationship(), ad.MemberOf, ad.MemberOfLocalGroup)}
-		coordC         = make(chan struct{}, analysis.MaximumDatabaseParallelWorkers)
 		traversalMap   = cardinality.ThreadSafeDuplex(cardinality.NewBitmap32())
+		traversalInst  = traversal.NewIDTraversal(db, analysis.MaximumDatabaseParallelWorkers)
 		memberships    = impact.NewThreadSafeAggregator(impact.NewIDA(func() cardinality.Provider[uint32] {
 			return cardinality.NewBitmap32()
 		}))
@@ -64,64 +64,46 @@ func ResolveAllGroupMemberships(ctx context.Context, db graph.Database, addition
 
 	log.Infof("Collected %d groups to resolve", len(adGroupIDs))
 
-	for i := 0; i < analysis.MaximumDatabaseParallelWorkers; i++ {
-		coordC <- struct{}{}
-	}
-
 	for _, adGroupID := range adGroupIDs {
 		if traversalMap.Contains(adGroupID.Uint32()) {
 			continue
 		}
 
-		<-coordC
-
-		go func(adGroupID graph.ID) {
-			if err := traversal.NewIDTraversal(db, analysis.MaximumDatabaseParallelWorkers).BreadthFirst(ctx, traversal.IDPlan{
-				Root: adGroupID,
-				Delegate: func(ctx context.Context, tx graph.Transaction, segment *graph.IDSegment) ([]*graph.IDSegment, error) {
-					if nextQuery, err := newTraversalQuery(tx, segment, graph.DirectionInbound, searchCriteria...); err != nil {
-						return nil, err
-					} else {
-						var nextSegments []*graph.IDSegment
-
-						if err := nextQuery.FetchTriples(func(cursor graph.Cursor[graph.RelationshipTripleResult]) error {
-							for nextTriple := range cursor.Chan() {
-								if traversalMap.CheckedAdd(nextTriple.StartID.Uint32()) {
-									nextSegments = append(nextSegments, segment.Descend(nextTriple.StartID, nextTriple.ID))
-								} else {
-									memberships.AddShortcut(segment.Descend(nextTriple.StartID, nextTriple.ID))
-								}
+		if err := traversalInst.BreadthFirst(ctx, traversal.IDPlan{
+			Root: adGroupID,
+			Delegate: func(ctx context.Context, tx graph.Transaction, segment *graph.IDSegment) ([]*graph.IDSegment, error) {
+				if nextQuery, err := newTraversalQuery(tx, segment, graph.DirectionInbound, searchCriteria...); err != nil {
+					return nil, err
+				} else {
+					var nextSegments []*graph.IDSegment
+
+					if err := nextQuery.FetchTriples(func(cursor graph.Cursor[graph.RelationshipTripleResult]) error {
+						for nextTriple := range cursor.Chan() {
+							if traversalMap.CheckedAdd(nextTriple.StartID.Uint32()) {
+								nextSegments = append(nextSegments, segment.Descend(nextTriple.StartID, nextTriple.ID))
+							} else {
+								memberships.AddShortcut(segment.Descend(nextTriple.StartID, nextTriple.ID))
 							}
-
-							return cursor.Error()
-						}); err != nil {
-							return nil, err
-						}
-
-						// Is this path terminal?
-						if len(nextSegments) == 0 {
-							memberships.AddPath(segment)
 						}
 
-						return nextSegments, nil
+						return cursor.Error()
+					}); err != nil {
+						return nil, err
 					}
-				},
-			}); err != nil {
-				log.Errorf("Error during traversal: %v", err)
-			}
-
-			coordC <- struct{}{}
-		}(adGroupID)
-	}
 
-	log.Infof("Finished submitting all groups to be traversed. Waiting...")
+					// Is this path terminal?
+					if len(nextSegments) == 0 {
+						memberships.AddPath(segment)
+					}
 
-	for finished := 0; finished < analysis.MaximumDatabaseParallelWorkers; {
-		<-coordC
-		finished++
+					return nextSegments, nil
+				}
+			},
+		}); err != nil {
+			return nil, err
+		}
 	}
 
-	close(coordC)
 	return memberships, nil
 }
 

packages/go/slicesext/slicesext_test.go

@@ -28,17 +28,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var (
-	listEmpty        = []int{}
-	listSingle       = []int{0}
-	listDuo          = []int{0, 1}
-	reversedListDuo  = []int{1, 0}
-	listEven         = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
-	reversedListEven = []int{15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}
-	listOdd          = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}
-	reversedListOdd  = []int{14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}
-)
-
 func TestFilter(t *testing.T) {
 	require.Equal(t, []int{1, 3}, slicesext.Filter([]int{1, 2, 3, 4}, isOdd))
 	require.Equal(t, []string{"bbbbbb", "cccccccc"}, slicesext.Filter([]string{"aaaa", "bbbbbb", "cccccccc", "dd"}, isLong))